From a523bdac8116382f9b61e8678e8b08cd8a5544b6 Mon Sep 17 00:00:00 2001 From: homelab-runtime-publisher Date: Wed, 23 Sep 2026 21:02:38 +0000 Subject: [PATCH] runtime: publish FORUM-260923-RCA-PVE02-VM160-AUDITC7A1R1 --- ...UM-260923-RCA-PVE02-VM160-AUDITC7A1R1.json | 38 + ...RUM-260923-RCA-PVE02-VM160-AUDITC7A1R1.txt | 291 ++ runtime/latest.json | 16 +- runtime/latest.txt | 2348 ++--------------- 4 files changed, 600 insertions(+), 2093 deletions(-) create mode 100644 runtime/history/FORUM-260923-RCA-PVE02-VM160-AUDITC7A1R1.json create mode 100644 runtime/history/FORUM-260923-RCA-PVE02-VM160-AUDITC7A1R1.txt diff --git a/runtime/history/FORUM-260923-RCA-PVE02-VM160-AUDITC7A1R1.json b/runtime/history/FORUM-260923-RCA-PVE02-VM160-AUDITC7A1R1.json new file mode 100644 index 00000000..92c90162 --- /dev/null +++ b/runtime/history/FORUM-260923-RCA-PVE02-VM160-AUDITC7A1R1.json @@ -0,0 +1,38 @@ +{ + "schema_version": 1, + "channel": "homelab-runtime", + "command_id": "FORUM-260923-RCA-PVE02-VM160-AUDITC7A1R1", + "status": "OK", + "rc": 0, + "host": "pve01", + "mode": "read-only", + "component": "forum-pve02-vm160-audit", + "started_at_utc": "2026-09-23T21:02:32Z", + "finished_at_utc": "2026-09-23T21:02:38Z", + "reference_register_checked": true, + "reference_sha256": "5a3d8e5154c41cb582a4e0aca68090be1f0138918bf82131a948df326f9d8d66", + "error_register_checked": true, + "error_register_sha256": "3b09a553ec0f527ed3afeed4753f52a74ee3036045bbfb3c685e9f8af4ba7ba0", + "command_sha256": "d0d58f5b0d26ba9465d13d1ce6a174370a5532296fef88f757fd4fb1f2152a12", + "duplicate_failed_command_blocked": false, + "block_reason": null, + "execution_started": true, + "change_declared": false, + "result_contract_valid": true, + "result_contract_status": null, + "result_contract_error": null, + "command_rc": 0, + "changes_made": false, + "rollback_started": false, + "rollback_restored": null, + "mutation_outcome": "NO_MUTATION", + "sanitized": true, + "secrets_included": false, + "private_addresses_included": false, + "raw_evidence_retained_locally": true, + "raw_evidence_sha256": "73f06c6c0cffab9eaddcd407da9d716944ca4ef348dd03eacfc4e7c8ceea1ea3", + "sanitized_output_sha256": "73f06c6c0cffab9eaddcd407da9d716944ca4ef348dd03eacfc4e7c8ceea1ea3", + "output_truncated_in_json": false, + "full_sanitized_output_url": "https://git.gram1.ru/.well-known/homelab-runtime/latest.txt", + "output": "CID=FORUM-260923-RCA-PVE02-VM160-AUDITC7A1R1\nMODE=read-only\nCOMPONENT=forum-pve02-vm160-audit\nMUTATION_BOUNDARY=NONE;PVE02_VM160_READ_ONLY;NO_MAIL_SEND;NO_DB_WRITE;NO_SERVICE_RESTART;NO_VM_CONFIG;NO_CLOUD\nEXECUTION_STARTED=true\nCHANGES_MADE=false\nROLLBACK_STARTED=false\nROLLBACK_RESTORED=null\nHANDOFF_SHA256=82a6060ccc216849c55790a3489ef157d5551a0c7cef0c5ec9abd35f569c7940\nPRIOR_CID=FORUM-260923-RCA-DEEP-AUDIT8D42R1\nPRIOR_FAILURE=VM160_NOT_RUNNING_ON_PVE01_WRONG_NODE\nCORRECTION=USE_PVE02_OWNER_NODE\nPVE02_TRANSPORT=PASS\nPVE02_EXPLICIT_KEY_USED=false\nPVE02_REMOTE_BEGIN=true\nPVE02_HOST=pve02\nVM160_STATE=running\nVM160_QGA_PING=OK\nVM160_QGA_TRANSPORT_RC=0\nVM160_GUEST_EXITCODE=0\nVM160_GUEST_AUDIT_BEGIN=true\nHOSTNAME=forum-prod\nTIME_UTC=2026-09-23T21:02:35Z\n=== SERVICES ===\nUNIT=nginx ACTIVE=active ENABLED=enabled\nUNIT=mariadb ACTIVE=active ENABLED=enabled\nUNIT=mysql ACTIVE=active ENABLED=alias\nUNIT=php8.3-fpm ACTIVE=active ENABLED=enabled\nUNIT=cron ACTIVE=active ENABLED=enabled\n=== FAILED UNITS ===\n=== CONFIG TESTS ===\nNGINX_CONFIGTEST_RC=0\nnginx: the configuration file /etc/nginx/nginx.conf syntax is ok\nnginx: configuration file /etc/nginx/nginx.conf test is successful\nPHPFPM_CONFIGTEST_BIN=php-fpm8.3 RC=0\n[23-Sep-2026 21:02:36] NOTICE: configuration file /etc/php/8.3/fpm/php-fpm.conf test is successful\n=== SENDMAIL/MSMTP STATE ===\nMAIL_BIN=/usr/sbin/sendmail mode=755 owner=root group=root size=143208\nMAIL_LINK=/usr/sbin/sendmail TARGET=../bin/msmtp\nMAIL_BIN=/usr/bin/msmtp mode=755 owner=root group=root size=143208\nMAIL_BIN_SHA256=d86ba2c0e12187ba16dcac322e4e94bd6079e635c1793b1690678b45bf3d0e61 PATH=/usr/bin/msmtp\nMAIL_BIN=/usr/local/sbin/codevipe-sendmail mode=755 owner=root group=root size=810\nMAIL_BIN_SHA256=e50ba5dcf4ac7a905ed1e6b1cc7c301a61318f9fe73976eeda3dd21047f39958 PATH=/usr/local/sbin/codevipe-sendmail\nMSMTP_CONFIG_META=/etc/msmtprc mode=640 owner=root group=xfp_mail size=319\nMSMTP_CONFIG_SHA256=c00fe15969542c6cda983e9b937488c9745e97e8fd8b2d75fdf6bd5386e722a9 PATH=/etc/msmtprc\nMSMTP_CONFIG_MISSING=/root/.msmtprc\nMSMTP_FILE_META=/etc/msmtp/pvepro.env mode=640 owner=root group=www-data size=150\nMSMTP_FILE_SHA256=32330a022cb6fbde19efb486c9945a6251070a544443a2947e7b222401a1ffee PATH=/etc/msmtp/pvepro.env\nsendmail_path => /usr/sbin/sendmail -t -i => /usr/sbin/sendmail -t -i\n/etc/php/8.3/fpm/pool.d/xf-dsmods.conf:2:php_admin_value[sendmail_path] = /usr/local/sbin/dsmods-sendmail -t -i\n/etc/php/8.3/fpm/pool.d/xf-kingofwolk.conf:2:php_admin_value[sendmail_path] = /usr/local/sbin/kingofwolk-sendmail -t -i\n/etc/php/8.3/fpm/pool.d/xf-codevipe.conf:2:php_admin_value[sendmail_path] = /usr/local/sbin/codevipe-sendmail -t -i\n/etc/php/8.3/fpm/pool.d/www.conf:486:;php_admin_value[sendmail_path] = /usr/sbin/sendmail -t -i -f \n/etc/php/8.3/fpm/pool.d/xf-zakrutim.conf:2:php_admin_value[sendmail_path] = /usr/local/sbin/zakrutim-sendmail -t -i\n/etc/php/8.3/fpm/pool.d/xf-hkmods.conf:2:php_admin_value[sendmail_path] = /usr/local/sbin/hkmods-sendmail -t -i\n/etc/php/8.3/fpm/pool.d/xf-gamevipe.conf:2:php_admin_value[sendmail_path] = /usr/local/sbin/gamevipe-sendmail -t -i\n=== MSMTP RECENT ERRORS ===\nMSMTP_LOG_PRESENT=/var/log/msmtp.log\nJul 01 07:19:46 host=mail.pvepro.ru tls=on auth=on user= from= recipients= smtpstatus=535 smtpmsg='535 5.7.8 Error: authentication failed: (reason unavailable)' errormsg='authentication failed (method PLAIN)' exitcode=EX_NOPERM\nJul 01 07:23:11 host=mail.pvepro.ru tls=on auth=on user= from= recipients= smtpstatus=535 smtpmsg='535 5.7.8 Error: authentication failed: (reason unavailable)' errormsg='authentication failed (method PLAIN)' exitcode=EX_NOPERM\nJul 01 07:23:19 host=mail.pvepro.ru tls=on auth=on user= from= recipients= smtpstatus=535 smtpmsg='535 5.7.8 Error: authentication failed: (reason unavailable)' errormsg='authentication failed (method PLAIN)' exitcode=EX_NOPERM\nJul 01 07:23:27 host=mail.pvepro.ru tls=on auth=on user= from= recipients= smtpstatus=535 smtpmsg='535 5.7.8 Error: authentication failed: (reason unavailable)' errormsg='authentication failed (method PLAIN)' exitcode=EX_NOPERM\nJul 01 08:08:27 host=mail.pvepro.ru tls=on auth=on user= from= recipients= mailsize=242 smtpstatus=250 smtpmsg='250 2.0.0 Ok: queued as 56D68A7F1C' exitcode=EX_OK\nJul 01 08:59:09 host=mail.pvepro.ru tls=on auth=on user= from= recipients= mailsize=1577 smtpstatus=250 smtpmsg='250 2.0.0 Ok: queued as 78667A73C3' exitcode=EX_OK\nJul 01 09:14:24 host=mail.pvepro.ru tls=on auth=on user= from= recipients= mailsize=1628 smtpstatus=250 smtpmsg='250 2.0.0 Ok: queued as D36DBA756C' exitcode=EX_OK\nJul 12 09:30:36 host=mail.pvepro.ru tls=on auth=on user= from= recipients= mailsize=7695 smtpstatus=250 smtpmsg='250 2.0.0 Ok: queued as 1166CA7FEF' exitcode=EX_OK\nJul 12 09:31:50 host=mail.pvepro.ru tls=on auth=on user= from= recipients= mailsize=7695 smtpstatus=250 smtpmsg='250 2.0.0 Ok: queued as 839D2A7FEF' exitcode=EX_OK\nJul 16 14:13:27 host=mail.pvepro.ru tls=on auth=on user= from= recipients= mailsize=7695 smtpstatus=250 smtpmsg='250 2.0.0 Ok: queued as 8E3D8A7FF9' exitcode=EX_OK\nJul 16 14:14:37 host=mail.pvepro.ru tls=on auth=on user= from= recipients= mailsize=7695 smtpstatus=250 smtpmsg='250 2.0.0 Ok: queued as 839E3A73F4' exitcode=EX_OK\nJul 16 14:15:19 host=mail.pvepro.ru tls=on auth=on user= from= recipients= mailsize=7695 smtpstatus=250 smtpmsg='250 2.0.0 Ok: queued as D1B4BA8012' exitcode=EX_OK\nJul 19 21:07:23 host=mail.pvepro.ru tls=on auth=on user= from= recipients= mailsize=7701 smtpstatus=250 smtpmsg='250 2.0.0 Ok: queued as A6FB3A8121' exitcode=EX_OK\nJul 22 11:29:09 host=mail.pvepro.ru tls=on auth=on user= from= recipients= mailsize=7707 smtpstatus=250 smtpmsg='250 2.0.0 Ok: queued as 77305A8015' exitcode=EX_OK\nJul 24 05:51:06 host=mail.pvepro.ru tls=on auth=on user= from= recipients= mailsize=7681 smtpstatus=250 smtpmsg='250 2.0.0 Ok: queued as ED00EA734F' exitcode=EX_OK\nJul 26 06:38:28 host=mail.pvepro.ru tls=on auth=on user= from= recipients= mailsize=7744 smtpstatus=250 smtpmsg='250 2.0.0 Ok: queued as 2DEA8A7F61' exitcode=EX_OK\nJul 28 19:04:01 host=mail.pvepro.ru tls=on auth=on user= from= recipients= mailsize=7697 smtpstatus=250 smtpmsg='250 2.0.0 Ok: queued as 5CE98A737C' exitcode=EX_OK\nJul 28 19:05:08 host=mail.pvepro.ru tls=on auth=on user= from= recipients= mailsize=7697 smtpstatus=250 smtpmsg='250 2.0.0 Ok: queued as E140AA737C' exitcode=EX_OK\nJul 29 08:48:58 host=mail.pvepro.ru tls=on auth=on user= from= recipients= mailsize=7669 smtpstatus=250 smtpmsg='250 2.0.0 Ok: queued as 227FBA7431' exitcode=EX_OK\nJul 29 20:33:47 host=mail.pvepro.ru tls=on auth=on user= from= recipients= mailsize=7685 smtpstatus=250 smtpmsg='250 2.0.0 Ok: queued as 9EBCFA7F0E' exitcode=EX_OK\nJul 30 16:48:00 host=mail.pvepro.ru tls=on auth=on user= from= recipients= mailsize=7687 smtpstatus=250 smtpmsg='250 2.0.0 Ok: queued as B1345A737C' exitcode=EX_OK\nAug 17 08:05:32 host=mail.pvepro.ru tls=on auth=on user= from= recipients= mailsize=7738 smtpstatus=250 smtpmsg='250 2.0.0 Ok: queued as 3EF03A7F8D' exitcode=EX_OK\nAug 17 08:07:52 host=mail.pvepro.ru tls=on auth=on user= from= recipients= mailsize=7738 smtpstatus=250 smtpmsg='250 2.0.0 Ok: queued as 6635BA73AA' exitcode=EX_OK\nAug 18 16:58:38 host=postbox.cloud.yandex.net tls=on auth=on user=ajeups2v3l2qfl9t1k9a from= recipients= mailsize=440 smtpstatus=250 smtpmsg='250 2.6.0 OK: queued as DKS89L8IPX05.3NYOOU5KRIBAX@ingress2-vla' exitcode=EX_OK\nAug 18 18:59:07 host=postbox.cloud.yandex.net tls=on auth=on user=ajeups2v3l2qfl9t1k9a from= recipients= mailsize=492 smtpstatus=250 smtpmsg='250 2.6.0 OK: queued as DKSATUF4C1XX.329CV6B95FH4R@ingress2-vla' exitcode=EX_OK\nAug 21 15:02:48 host=postbox.cloud.yandex.net tls=on auth=on user=ajeups2v3l2qfl9t1k9a from= recipients= mailsize=7672 smtpstatus=250 smtpmsg='250 2.6.0 OK: queued as DKUPOJOWNNPA.3CH5IJH2A8PW1@ingress2-klg' exitcode=EX_OK\nSep 02 19:39:11 host=postbox.cloud.yandex.net tls=on auth=on user=ajeups2v3l2qfl9t1k9a from= recipients= smtpstatus=550 smtpmsg='550 5.4.1 identity not found RequestID: 488b75b4-c264-42eb-b6fc-0cae6b175a7a' errormsg='envelope from address not accepted by the server' exitcode=EX_DATAERR\nSep 02 19:43:53 host=smtp.yandex.ru tls=on auth=on user= from= recipients= mailsize=257 smtpstatus=250 smtpmsg='250 2.0.0 Ok: queued on mail-nwsmtp-smtp-production-main-95.klg.yp-c.yandex.net 1788378233-qhqjKj2mO8c0-EjXDwBNJ' exitcode=EX_OK\nSep 02 19:44:53 host=smtp.yandex.ru tls=on auth=on user= from= recipients= mailsize=254 smtpstatus=250 smtpmsg='250 2.0.0 Ok: queued on mail-nwsmtp-smtp-production-main-71.sas.yp-c.yandex.net 1788378293-riqme1Oc6uQ0-JA3QSjr3' exitcode=EX_OK\nSep 02 21:33:45 host=smtp.yandex.ru tls=on auth=on user= from= recipients= mailsize=7477 smtpstatus=250 smtpmsg='250 2.0.0 Ok: queued on mail-nwsmtp-smtp-production-main-99.klg.yp-c.yandex.net 1788384825-iXsB0HPeHeA0-t67v0c58' exitcode=EX_OK\nSep 03 04:35:14 host=smtp.yandex.ru tls=on auth=on user= from= recipients= mailsize=2390 smtpstatus=250 smtpmsg='250 2.0.0 Ok: queued on mail-nwsmtp-smtp-production-main-85.iva.yp-c.yandex.net 1788410114-DZclo65g6Sw0-ym39kyd6' exitcode=EX_OK\nSep 03 04:44:35 host=smtp.yandex.ru tls=on auth=on user= from= recipients= mailsize=2412 smtpstatus=250 smtpmsg='250 2.0.0 Ok: queued on mail-nwsmtp-smtp-production-main-89.sas.yp-c.yandex.net 1788410675-ZicUTjSgH4Y0-ino0SiK0' exitcode=EX_OK\nSep 04 04:09:17 host=smtp.yandex.ru tls=on auth=on user= from= recipients= mailsize=7474 smtpstatus=250 smtpmsg='250 2.0.0 Ok: queued on mail-nwsmtp-smtp-production-main-88.klg.yp-c.yandex.net 1788494957-H9Anioed9mI0-YAPrVYsC' exitcode=EX_OK\nSep 04 04:09:44 host=smtp.yandex.ru tls=on auth=on user= from= recipients= mailsize=20404 smtpstatus=250 smtpmsg='250 2.0.0 Ok: queued on mail-nwsmtp-smtp-production-main-59.klg.yp-c.yandex.net 1788494984-h9AZQbdd08c0-OXrP5vgQ' exitcode=EX_OK\nSep 04 04:19:31 host=smtp.yandex.ru tls=on auth=on user= from= recipients= mailsize=7632 smtpstatus=250 smtpmsg='250 2.0.0 Ok: queued on mail-nwsmtp-smtp-production-main-98.sas.yp-c.yandex.net 1788495571-UJAjYwgdJCg0-xBGi7vHN' exitcode=EX_OK\nSep 04 06:00:34 host=smtp.yandex.ru tls=on auth=on user= from= recipients= mailsize=7703 smtpstatus=250 smtpmsg='250 2.0.0 Ok: queued on mail-nwsmtp-smtp-production-main-59.iva.yp-c.yandex.net 1788501634-X0CkCDGeJqM0-ewqs8N37' exitcode=EX_OK\nSep 08 08:25:56 host=smtp.yandex.ru tls=on auth=on user= from= recipients= mailsize=7704 smtpstatus=250 smtpmsg='250 2.0.0 Ok: queued on mail-nwsmtp-smtp-production-main-89.sas.yp-c.yandex.net 1788855956-tPIpFURgCSw0-0bD0dt06' exitcode=EX_OK\nSep 10 00:10:02 host=smtp.yandex.ru tls=on auth=on user= from= recipients= smtpstatus=553 smtpmsg='553 5.7.1 Sender address rejected: user not found. 1788999002-1ACLU4egAmI0-BvLQE2bE' errormsg='envelope from address not accepted by the server' exitcode=EX_DATAERR\nSep 10 00:10:03 host=smtp.yandex.ru tls=on auth=on user= from= recipients= mailsize=10460 smtpstatus=250 smtpmsg='250 2.0.0 Ok: queued on mail-nwsmtp-smtp-production-main-71.sas.yp-c.yandex.net 1788999003-3ACiEWkcAmI0-sgTWLJGA' exitcode=EX_OK\nSep 10 00:10:05 host=smtp.yandex.ru tls=on auth=on user= from= recipients= mailsize=10599 smtpstatus=250 smtpmsg='250 2.0.0 Ok: queued on mail-nwsmtp-smtp-production-main-60.sas.yp-c.yandex.net 1788999005-4ACnvojfQa60-IhiFG02U' exitcode=EX_OK\nSep 10 00:10:06 host=smtp.yandex.ru tls=on auth=on user= from= recipients= smtpstatus=553 smtpmsg='553 5.7.1 Sender address rejected: user not found. 1788999006-6ACgTJhfxuQ0-ykq2gT2m' errormsg='envelope from address not accepted by the server' exitcode=EX_DATAERR\nSep 10 00:10:07 host=smtp.yandex.ru tls=on auth=on user= from= recipients= smtpstatus=553 smtpmsg='553 5.7.1 Sender address rejected: user not found. 1788999007-7ACYnudg6mI0-wrzZ9ta2' errormsg='envelope from address not accepted by the server' exitcode=EX_DATAERR\nSep 10 00:10:08 host=smtp.yandex.ru tls=on auth=on user= from= recipients= mailsize=10224 smtpstatus=250 smtpmsg='250 2.0.0 Ok: queued on mail-nwsmtp-smtp-production-main-58.iva.yp-c.yandex.net 1788999008-8ACm4u3oe8c0-EAVVXjUV' exitcode=EX_OK\nSep 13 16:14:26 host=smtp.yandex.ru tls=on auth=on user= from= recipients= mailsize=7702 smtpstatus=250 smtpmsg='250 2.0.0 Ok: queued on mail-nwsmtp-smtp-production-main-67.vla.yp-c.yandex.net 1789316066-QEVwdXHe9Os0-PH6Ad0od' exitcode=EX_OK\nSep 19 00:10:02 host=smtp.yandex.ru tls=on auth=on user= from= recipients= smtpstatus=553 smtpmsg='553 5.7.1 Sender address rejected: user not found. 1789776602-2ALqagFfpGk0-vq5tlHNF' errormsg='envelope from address not accepted by the server' exitcode=EX_DATAERR\nSep 19 00:10:03 host=smtp.yandex.ru tls=on auth=on user= from= recipients= smtpstatus=553 smtpmsg='553 5.7.1 Sender address rejected: user not found. 1789776603-3ALZ9cFg74Y0-Rbt8yJgu' errormsg='envelope from address not accepted by the server' exitcode=EX_DATAERR\nSep 19 00:10:04 host=smtp.yandex.ru tls=on auth=on user= from= recipients= smtpstatus=553 smtpmsg='553 5.7.1 Sender address rejected: user not found. 1789776604-4AL5ABFdSSw0-XwXfZVbq' errormsg='envelope from address not accepted by the server' exitcode=EX_DATAERR\nSep 19 00:10:05 host=smtp.yandex.ru tls=on auth=on user= from= recipients= smtpstatus=553 smtpmsg='553 5.7.1 Sender address rejected: user not found. 1789776605-5ALSb5FdeKo0-ugaNiHmf' errormsg='envelope from address not accepted by the server' exitcode=EX_DATAERR\nSep 19 00:10:06 host=smtp.yandex.ru tls=on auth=on user= from= recipients= smtpstatus=553 smtpmsg='553 5.7.1 Sender address rejected: user not found. 1789776606-6ALAYdFdNuQ0-dL2d7DHS' errormsg='envelope from address not accepted by the server' exitcode=EX_DATAERR\nSep 19 00:10:07 host=smtp.yandex.ru tls=on auth=on user= from= recipients= smtpstatus=553 smtpmsg='553 5.7.1 Sender address rejected: user not found. 1789776607-7ALu7bFe4Gk0-WcjF4jNs' errormsg='envelope from address not accepted by the server' exitcode=EX_DATAERR\n=== JOB RUNNER ===\nJOBRUNNER_META=/root/scripts/forum-xenforo-run-jobs.sh mode=700 owner=root group=root size=2343\nJOBRUNNER_SHA256=5cc6c6cba317e0f0c4b1b13f375d8677b3c9c96d0416f998315a7d4e05e8a1aa PATH=/root/scripts/forum-xenforo-run-jobs.sh\nJOBRUNNER_META=/etc/cron.d/forum-xenforo-run-jobs mode=644 owner=root group=root size=183\nJOBRUNNER_SHA256=20cf9e4d059148c14094d145e778bb307b21183f608ff9b430dd620b9ece75c4 PATH=/etc/cron.d/forum-xenforo-run-jobs\n=== XENFORO SITES ===\nXENFORO_ROOT_COUNT=6\nSITE_BEGIN=1\nROOT_SHA256=90d10058f0574d8a5ff1ff412fb7b6a1eab2119d09c8b8162a0f8ca55d15180f\nDB_CONNECT=OK\nBOARD_DOMAIN=codevipe.ru\nEMAIL_TRANSPORT={\"emailTransport\":\"sendmail\"}\nDEFAULTEMAILADDRESS_PRESENT=true\nDEFAULTEMAILADDRESS_SHA256=2d4e7d3cf84ec1ba079400cf62ca99e7464a221a686ebe73405b73fb01ffe6f4\nBOUNCEEMAILADDRESS_PRESENT=true\nBOUNCEEMAILADDRESS_SHA256=1b5fc8a44aa74c2f02b7ce893f8a0e0a901d60b3f5e68b39c9448e7265117265\nCONTACTEMAILADDRESS_PRESENT=true\nCONTACTEMAILADDRESS_SHA256=0d4cf60ca8cd53a12ad50dc5419d3c9c32192f0852000eb10051fbdef52b04bf\nXF_ERROR_COUNT_24H=0\nXF_ERROR_COUNT_7D=2\nXF_ERROR_COUNT_30D=2\nXF_ERROR_COUNT_ALL=2\nXF_ERROR_TYPE=XF\\Db\\InvalidQueryException COUNT=1 LAST=1789960536\nXF_ERROR_TYPE=Symfony\\Component\\Mailer\\Exception\\TransportException COUNT=1 LAST=1789776602\nXF_ERROR_ROW ID=2 TS=1789960536 TYPE=XF\\Db\\InvalidQueryException FILE=AbstractStatement.php LINE=225 MSG=MySQL statement prepare error [1064]: You have an error in your SQL syntax; check the manual that corresponds to your MariaDB server version for the right syntax to use near 'FROM xf_template_modification WHERE addon_id='Gram/Support' ORDER MSG_SHA256=9ad3a33fd2ebdd213c22e2156d0358b02f9f6f82e2027f3101055bbfe5dd83b5\nXF_ERROR_ROW ID=1 TS=1789776602 TYPE=Symfony\\Component\\Mailer\\Exception\\TransportException FILE=ProcessStream.php LINE=74 MSG=Email to from failed: Process failed with exit code 65: sendmail: envelope from address not accepted by the server sendmail: server message: 553 5.7.1 Sender address rejected: user not found. 1789776602-2ALqagFfpGk0- MSG_SHA256=6ec17ccf28a8f55861f6ceb88ab722165bd922e90a12976f2755bc9db10c3638\nXF_MAIL_QUEUE_TABLE=MISSING\nXF_JOB_COUNT=2\nSITE_PHP_AUDIT_RC=0\nSITE_END=1\nSITE_BEGIN=2\nROOT_SHA256=d33b4c353c031284c61f9632f49971cd1f833ab4f48c0da27e4fdf81c373652e\nDB_CONNECT=OK\nBOARD_DOMAIN=dsmods.ru\nEMAIL_TRANSPORT={\"emailTransport\":\"sendmail\"}\nDEFAULTEMAILADDRESS_PRESENT=true\nDEFAULTEMAILADDRESS_SHA256=953017f372fc160381792c8f72822095c821e5f0c4519776c44cfa01fb6c16f1\nBOUNCEEMAILADDRESS_PRESENT=true\nBOUNCEEMAILADDRESS_SHA256=ba70f629ed27c307125a0538ca1c95049ac153aea2b21a474afeb4e3ad60d08d\nCONTACTEMAILADDRESS_PRESENT=true\nCONTACTEMAILADDRESS_SHA256=0d4cf60ca8cd53a12ad50dc5419d3c9c32192f0852000eb10051fbdef52b04bf\nXF_ERROR_COUNT_24H=0\nXF_ERROR_COUNT_7D=2\nXF_ERROR_COUNT_30D=2\nXF_ERROR_COUNT_ALL=2\nXF_ERROR_TYPE=XF\\Db\\InvalidQueryException COUNT=1 LAST=1789960537\nXF_ERROR_TYPE=Symfony\\Component\\Mailer\\Exception\\TransportException COUNT=1 LAST=1789776606\nXF_ERROR_ROW ID=2 TS=1789960537 TYPE=XF\\Db\\InvalidQueryException FILE=AbstractStatement.php LINE=225 MSG=MySQL statement prepare error [1064]: You have an error in your SQL syntax; check the manual that corresponds to your MariaDB server version for the right syntax to use near 'FROM xf_template_modification WHERE addon_id='Gram/Support' ORDER MSG_SHA256=9ad3a33fd2ebdd213c22e2156d0358b02f9f6f82e2027f3101055bbfe5dd83b5\nXF_ERROR_ROW ID=1 TS=1789776606 TYPE=Symfony\\Component\\Mailer\\Exception\\TransportException FILE=ProcessStream.php LINE=74 MSG=Email to from failed: Process failed with exit code 65: sendmail: envelope from address not accepted by the server sendmail: server message: 553 5.7.1 Sender address rejected: user not found. 1789776606-6ALAYdFdNuQ0- MSG_SHA256=59e1133715aedc5b530f2a2f87f31d659af13424d86fdc7b06d1331635178a0c\nXF_MAIL_QUEUE_TABLE=MISSING\nXF_JOB_COUNT=2\nSITE_PHP_AUDIT_RC=0\nSITE_END=2\nSITE_BEGIN=3\nROOT_SHA256=e06523a33d0b38a0ec6532783b5e63542eeba1303e68f2efad63214fd1e8908f\nDB_CONNECT=OK\nBOARD_DOMAIN=gamevipe.ru\nEMAIL_TRANSPORT={\"emailTransport\":\"sendmail\"}\nDEFAULTEMAILADDRESS_PRESENT=true\nDEFAULTEMAILADDRESS_SHA256=ded102c05ab32e41f2ada4d15b63c6adb848e14e76154015f294b751459a75dc\nBOUNCEEMAILADDRESS_PRESENT=true\nBOUNCEEMAILADDRESS_SHA256=3c6809f1cc3dd12044ad384b8d944a408bd84b4320af0bb505a5fb5ebf3f10e3\nCONTACTEMAILADDRESS_PRESENT=true\nCONTACTEMAILADDRESS_SHA256=0d4cf60ca8cd53a12ad50dc5419d3c9c32192f0852000eb10051fbdef52b04bf\nXF_ERROR_COUNT_24H=0\nXF_ERROR_COUNT_7D=1\nXF_ERROR_COUNT_30D=1\nXF_ERROR_COUNT_ALL=1\nXF_ERROR_TYPE=Symfony\\Component\\Mailer\\Exception\\TransportException COUNT=1 LAST=1789776603\nXF_ERROR_ROW ID=1 TS=1789776603 TYPE=Symfony\\Component\\Mailer\\Exception\\TransportException FILE=ProcessStream.php LINE=74 MSG=Email to from failed: Process failed with exit code 65: sendmail: envelope from address not accepted by the server sendmail: server message: 553 5.7.1 Sender address rejected: user not found. 1789776603-3ALZ9cFg74Y0- MSG_SHA256=f0aeb49dcba2c41db1f69663611a12e57913ca0b631287cdcc3c3b2a8db73f51\nXF_MAIL_QUEUE_TABLE=MISSING\nXF_JOB_COUNT=3\nSITE_PHP_AUDIT_RC=0\nSITE_END=3\nSITE_BEGIN=4\nROOT_SHA256=127c45667b0b6b6f23d20d6cc66d094b61a854801e42025ad2cb81ab5bfbc03e\nDB_CONNECT=OK\nBOARD_DOMAIN=hkmods.ru\nEMAIL_TRANSPORT={\"emailTransport\":\"sendmail\"}\nDEFAULTEMAILADDRESS_PRESENT=true\nDEFAULTEMAILADDRESS_SHA256=ef08c646d1d1fc56ef350fba44e29f2117996462bf46d9d14fa33c0688f9f4e3\nBOUNCEEMAILADDRESS_PRESENT=true\nBOUNCEEMAILADDRESS_SHA256=0097bdfb68ae6258a87c12c8dae4823421d4209086a2f190cb189defbda96bc3\nCONTACTEMAILADDRESS_PRESENT=true\nCONTACTEMAILADDRESS_SHA256=0d4cf60ca8cd53a12ad50dc5419d3c9c32192f0852000eb10051fbdef52b04bf\nXF_ERROR_COUNT_24H=0\nXF_ERROR_COUNT_7D=1\nXF_ERROR_COUNT_30D=1\nXF_ERROR_COUNT_ALL=1\nXF_ERROR_TYPE=Symfony\\Component\\Mailer\\Exception\\TransportException COUNT=1 LAST=1789776604\nXF_ERROR_ROW ID=1 TS=1789776604 TYPE=Symfony\\Component\\Mailer\\Exception\\TransportException FILE=ProcessStream.php LINE=74 MSG=Email to from failed: Process failed with exit code 65: sendmail: envelope from address not accepted by the server sendmail: server message: 553 5.7.1 Sender address rejected: user not found. 1789776604-4AL5ABFdSSw0- MSG_SHA256=03dcb6ecd92e29dde8dcccc9776bb4af1985d7d525bfae5cf9b5bb4db579de95\nXF_MAIL_QUEUE_TABLE=MISSING\nXF_JOB_COUNT=3\nSITE_PHP_AUDIT_RC=0\nSITE_END=4\nSITE_BEGIN=5\nROOT_SHA256=d249590aad8f87daf95d1fed623405edb307aa6c9b8123b2aaf89da88fc117ab\nDB_CONNECT=OK\nBOARD_DOMAIN=kingofwolk.ru\nEMAIL_TRANSPORT={\"emailTransport\":\"sendmail\"}\nDEFAULTEMAILADDRESS_PRESENT=true\nDEFAULTEMAILADDRESS_SHA256=e3b5d95ee979b3970d8ac7a07e503a6f5247963b701b1d1ad0b4aca12fc41d3a\nBOUNCEEMAILADDRESS_PRESENT=true\nBOUNCEEMAILADDRESS_SHA256=e49d0b3b9f084cf1223dd2b108019277e2b51caf2cf1a63b3ba3853765aefc65\nCONTACTEMAILADDRESS_PRESENT=true\nCONTACTEMAILADDRESS_SHA256=0d4cf60ca8cd53a12ad50dc5419d3c9c32192f0852000eb10051fbdef52b04bf\nXF_ERROR_COUNT_24H=0\nXF_ERROR_COUNT_7D=1\nXF_ERROR_COUNT_30D=1\nXF_ERROR_COUNT_ALL=1\nXF_ERROR_TYPE=Symfony\\Component\\Mailer\\Exception\\TransportException COUNT=1 LAST=1789776607\nXF_ERROR_ROW ID=1 TS=1789776607 TYPE=Symfony\\Component\\Mailer\\Exception\\TransportException FILE=ProcessStream.php LINE=74 MSG=Email to from failed: Process failed with exit code 65: sendmail: envelope from address not accepted by the server sendmail: server message: 553 5.7.1 Sender address rejected: user not found. 1789776607-7ALu7bFe4Gk0- MSG_SHA256=075f2b1963657d083c11e214ecc8cfd4e01d30f9ac1723bc440808730e6fb809\nXF_MAIL_QUEUE_TABLE=MISSING\nXF_JOB_COUNT=2\nSITE_PHP_AUDIT_RC=0\nSITE_END=5\nSITE_BEGIN=6\nROOT_SHA256=b5bc9e40848a038adb66a5656917858172c1b70a14ccbe680e9d8f2260dda046\nDB_CONNECT=OK\nBOARD_DOMAIN=zakrutim.ru\nEMAIL_TRANSPORT={\"emailTransport\":\"sendmail\"}\nDEFAULTEMAILADDRESS_PRESENT=true\nDEFAULTEMAILADDRESS_SHA256=b719f473d6ff99139fef2749e636437b76aaf482963e82dde4ac69a296a2eab9\nBOUNCEEMAILADDRESS_PRESENT=true\nBOUNCEEMAILADDRESS_SHA256=a93aa0f45435a0e2c159c501fbe3cc70deaff82401d29d87a972e633f91df83a\nCONTACTEMAILADDRESS_PRESENT=true\nCONTACTEMAILADDRESS_SHA256=0d4cf60ca8cd53a12ad50dc5419d3c9c32192f0852000eb10051fbdef52b04bf\nXF_ERROR_COUNT_24H=0\nXF_ERROR_COUNT_7D=1\nXF_ERROR_COUNT_30D=1\nXF_ERROR_COUNT_ALL=1\nXF_ERROR_TYPE=Symfony\\Component\\Mailer\\Exception\\TransportException COUNT=1 LAST=1789776605\nXF_ERROR_ROW ID=1 TS=1789776605 TYPE=Symfony\\Component\\Mailer\\Exception\\TransportException FILE=ProcessStream.php LINE=74 MSG=Email to from failed: Process failed with exit code 65: sendmail: envelope from address not accepted by the server sendmail: server message: 553 5.7.1 Sender address rejected: user not found. 1789776605-5ALSb5FdeKo0- MSG_SHA256=894c6c5636182853711e897fbcf34d5881646135b6dffd7360737376e789d384\nXF_MAIL_QUEUE_TABLE=MISSING\nXF_JOB_COUNT=2\nSITE_PHP_AUDIT_RC=0\nSITE_END=6\n=== RECENT SERVICE ERROR COUNTS ===\nJOURNAL_ERROR_COUNT_24H_nginx=0\nJOURNAL_ERROR_COUNT_24H_mariadb=0\nJOURNAL_ERROR_COUNT_24H_mysql=0\nJOURNAL_ERROR_COUNT_24H_php8_3_fpm=0\nVM160_GUEST_AUDIT_END=true\nVM160_GUEST_STDERR_PRESENT=false\nVM160_QGA_PARSE_RC=0\nPVE02_REMOTE_END=true\nFAILURE=NONE\nDECISION=PASS_PVE02_VM160_AUDIT\nNEXT_GATE=REVIEW_FOR_BOUNDED_REPAIR\nTASK_RESULT=READ_ONLY_PVE02_VM160_FORUM_AUDIT_COMPLETE\nHOMELAB_RESULT_CONTRACT={\"version\":1,\"command_id\":\"FORUM-260923-RCA-PVE02-VM160-AUDITC7A1R1\",\"status\":\"OK\",\"changes_made\":false,\"rollback_started\":false,\"rollback_restored\":null}\n" +} diff --git a/runtime/history/FORUM-260923-RCA-PVE02-VM160-AUDITC7A1R1.txt b/runtime/history/FORUM-260923-RCA-PVE02-VM160-AUDITC7A1R1.txt new file mode 100644 index 00000000..cb4d5b49 --- /dev/null +++ b/runtime/history/FORUM-260923-RCA-PVE02-VM160-AUDITC7A1R1.txt @@ -0,0 +1,291 @@ +CHAT_OUTPUT_BEGIN +COMMAND_ID=FORUM-260923-RCA-PVE02-VM160-AUDITC7A1R1 +STATUS=OK +RC=0 +HOST=pve01 +MODE=read-only +COMPONENT=forum-pve02-vm160-audit +REFERENCE_REGISTER_CHECK=OK +REFERENCE_SHA256=5a3d8e5154c41cb582a4e0aca68090be1f0138918bf82131a948df326f9d8d66 +ERROR_REGISTER_CHECK=OK +ERROR_REGISTER_SHA256=3b09a553ec0f527ed3afeed4753f52a74ee3036045bbfb3c685e9f8af4ba7ba0 +COMMAND_SHA256=d0d58f5b0d26ba9465d13d1ce6a174370a5532296fef88f757fd4fb1f2152a12 +DUPLICATE_FAILED_COMMAND_BLOCKED=false +EXECUTION_STARTED=true +CHANGE_DECLARED=false +RESULT_CONTRACT_VALID=true +RESULT_CONTRACT_STATUS=NOT_APPLICABLE +RESULT_CONTRACT_ERROR=NONE +COMMAND_RC=0 +CHANGES_MADE=false +ROLLBACK_STARTED=false +ROLLBACK_RESTORED=null +MUTATION_OUTCOME=NO_MUTATION +SANITIZED=yes +SECRETS_INCLUDED=no +PRIVATE_ADDRESSES_INCLUDED=no +RAW_EVIDENCE_SHA256=73f06c6c0cffab9eaddcd407da9d716944ca4ef348dd03eacfc4e7c8ceea1ea3 +SANITIZED_OUTPUT_SHA256=73f06c6c0cffab9eaddcd407da9d716944ca4ef348dd03eacfc4e7c8ceea1ea3 +OUTPUT_BEGIN +CID=FORUM-260923-RCA-PVE02-VM160-AUDITC7A1R1 +MODE=read-only +COMPONENT=forum-pve02-vm160-audit +MUTATION_BOUNDARY=NONE;PVE02_VM160_READ_ONLY;NO_MAIL_SEND;NO_DB_WRITE;NO_SERVICE_RESTART;NO_VM_CONFIG;NO_CLOUD +EXECUTION_STARTED=true +CHANGES_MADE=false +ROLLBACK_STARTED=false +ROLLBACK_RESTORED=null +HANDOFF_SHA256=82a6060ccc216849c55790a3489ef157d5551a0c7cef0c5ec9abd35f569c7940 +PRIOR_CID=FORUM-260923-RCA-DEEP-AUDIT8D42R1 +PRIOR_FAILURE=VM160_NOT_RUNNING_ON_PVE01_WRONG_NODE +CORRECTION=USE_PVE02_OWNER_NODE +PVE02_TRANSPORT=PASS +PVE02_EXPLICIT_KEY_USED=false +PVE02_REMOTE_BEGIN=true +PVE02_HOST=pve02 +VM160_STATE=running +VM160_QGA_PING=OK +VM160_QGA_TRANSPORT_RC=0 +VM160_GUEST_EXITCODE=0 +VM160_GUEST_AUDIT_BEGIN=true +HOSTNAME=forum-prod +TIME_UTC=2026-09-23T21:02:35Z +=== SERVICES === +UNIT=nginx ACTIVE=active ENABLED=enabled +UNIT=mariadb ACTIVE=active ENABLED=enabled +UNIT=mysql ACTIVE=active ENABLED=alias +UNIT=php8.3-fpm ACTIVE=active ENABLED=enabled +UNIT=cron ACTIVE=active ENABLED=enabled +=== FAILED UNITS === +=== CONFIG TESTS === +NGINX_CONFIGTEST_RC=0 +nginx: the configuration file /etc/nginx/nginx.conf syntax is ok +nginx: configuration file /etc/nginx/nginx.conf test is successful +PHPFPM_CONFIGTEST_BIN=php-fpm8.3 RC=0 +[23-Sep-2026 21:02:36] NOTICE: configuration file /etc/php/8.3/fpm/php-fpm.conf test is successful +=== SENDMAIL/MSMTP STATE === +MAIL_BIN=/usr/sbin/sendmail mode=755 owner=root group=root size=143208 +MAIL_LINK=/usr/sbin/sendmail TARGET=../bin/msmtp +MAIL_BIN=/usr/bin/msmtp mode=755 owner=root group=root size=143208 +MAIL_BIN_SHA256=d86ba2c0e12187ba16dcac322e4e94bd6079e635c1793b1690678b45bf3d0e61 PATH=/usr/bin/msmtp +MAIL_BIN=/usr/local/sbin/codevipe-sendmail mode=755 owner=root group=root size=810 +MAIL_BIN_SHA256=e50ba5dcf4ac7a905ed1e6b1cc7c301a61318f9fe73976eeda3dd21047f39958 PATH=/usr/local/sbin/codevipe-sendmail +MSMTP_CONFIG_META=/etc/msmtprc mode=640 owner=root group=xfp_mail size=319 +MSMTP_CONFIG_SHA256=c00fe15969542c6cda983e9b937488c9745e97e8fd8b2d75fdf6bd5386e722a9 PATH=/etc/msmtprc +MSMTP_CONFIG_MISSING=/root/.msmtprc +MSMTP_FILE_META=/etc/msmtp/pvepro.env mode=640 owner=root group=www-data size=150 +MSMTP_FILE_SHA256=32330a022cb6fbde19efb486c9945a6251070a544443a2947e7b222401a1ffee PATH=/etc/msmtp/pvepro.env +sendmail_path => /usr/sbin/sendmail -t -i => /usr/sbin/sendmail -t -i +/etc/php/8.3/fpm/pool.d/xf-dsmods.conf:2:php_admin_value[sendmail_path] = /usr/local/sbin/dsmods-sendmail -t -i +/etc/php/8.3/fpm/pool.d/xf-kingofwolk.conf:2:php_admin_value[sendmail_path] = /usr/local/sbin/kingofwolk-sendmail -t -i +/etc/php/8.3/fpm/pool.d/xf-codevipe.conf:2:php_admin_value[sendmail_path] = /usr/local/sbin/codevipe-sendmail -t -i +/etc/php/8.3/fpm/pool.d/www.conf:486:;php_admin_value[sendmail_path] = /usr/sbin/sendmail -t -i -f +/etc/php/8.3/fpm/pool.d/xf-zakrutim.conf:2:php_admin_value[sendmail_path] = /usr/local/sbin/zakrutim-sendmail -t -i +/etc/php/8.3/fpm/pool.d/xf-hkmods.conf:2:php_admin_value[sendmail_path] = /usr/local/sbin/hkmods-sendmail -t -i +/etc/php/8.3/fpm/pool.d/xf-gamevipe.conf:2:php_admin_value[sendmail_path] = /usr/local/sbin/gamevipe-sendmail -t -i +=== MSMTP RECENT ERRORS === +MSMTP_LOG_PRESENT=/var/log/msmtp.log +Jul 01 07:19:46 host=mail.pvepro.ru tls=on auth=on user= from= recipients= smtpstatus=535 smtpmsg='535 5.7.8 Error: authentication failed: (reason unavailable)' errormsg='authentication failed (method PLAIN)' exitcode=EX_NOPERM +Jul 01 07:23:11 host=mail.pvepro.ru tls=on auth=on user= from= recipients= smtpstatus=535 smtpmsg='535 5.7.8 Error: authentication failed: (reason unavailable)' errormsg='authentication failed (method PLAIN)' exitcode=EX_NOPERM +Jul 01 07:23:19 host=mail.pvepro.ru tls=on auth=on user= from= recipients= smtpstatus=535 smtpmsg='535 5.7.8 Error: authentication failed: (reason unavailable)' errormsg='authentication failed (method PLAIN)' exitcode=EX_NOPERM +Jul 01 07:23:27 host=mail.pvepro.ru tls=on auth=on user= from= recipients= smtpstatus=535 smtpmsg='535 5.7.8 Error: authentication failed: (reason unavailable)' errormsg='authentication failed (method PLAIN)' exitcode=EX_NOPERM +Jul 01 08:08:27 host=mail.pvepro.ru tls=on auth=on user= from= recipients= mailsize=242 smtpstatus=250 smtpmsg='250 2.0.0 Ok: queued as 56D68A7F1C' exitcode=EX_OK +Jul 01 08:59:09 host=mail.pvepro.ru tls=on auth=on user= from= recipients= mailsize=1577 smtpstatus=250 smtpmsg='250 2.0.0 Ok: queued as 78667A73C3' exitcode=EX_OK +Jul 01 09:14:24 host=mail.pvepro.ru tls=on auth=on user= from= recipients= mailsize=1628 smtpstatus=250 smtpmsg='250 2.0.0 Ok: queued as D36DBA756C' exitcode=EX_OK +Jul 12 09:30:36 host=mail.pvepro.ru tls=on auth=on user= from= recipients= mailsize=7695 smtpstatus=250 smtpmsg='250 2.0.0 Ok: queued as 1166CA7FEF' exitcode=EX_OK +Jul 12 09:31:50 host=mail.pvepro.ru tls=on auth=on user= from= recipients= mailsize=7695 smtpstatus=250 smtpmsg='250 2.0.0 Ok: queued as 839D2A7FEF' exitcode=EX_OK +Jul 16 14:13:27 host=mail.pvepro.ru tls=on auth=on user= from= recipients= mailsize=7695 smtpstatus=250 smtpmsg='250 2.0.0 Ok: queued as 8E3D8A7FF9' exitcode=EX_OK +Jul 16 14:14:37 host=mail.pvepro.ru tls=on auth=on user= from= recipients= mailsize=7695 smtpstatus=250 smtpmsg='250 2.0.0 Ok: queued as 839E3A73F4' exitcode=EX_OK +Jul 16 14:15:19 host=mail.pvepro.ru tls=on auth=on user= from= recipients= mailsize=7695 smtpstatus=250 smtpmsg='250 2.0.0 Ok: queued as D1B4BA8012' exitcode=EX_OK +Jul 19 21:07:23 host=mail.pvepro.ru tls=on auth=on user= from= recipients= mailsize=7701 smtpstatus=250 smtpmsg='250 2.0.0 Ok: queued as A6FB3A8121' exitcode=EX_OK +Jul 22 11:29:09 host=mail.pvepro.ru tls=on auth=on user= from= recipients= mailsize=7707 smtpstatus=250 smtpmsg='250 2.0.0 Ok: queued as 77305A8015' exitcode=EX_OK +Jul 24 05:51:06 host=mail.pvepro.ru tls=on auth=on user= from= recipients= mailsize=7681 smtpstatus=250 smtpmsg='250 2.0.0 Ok: queued as ED00EA734F' exitcode=EX_OK +Jul 26 06:38:28 host=mail.pvepro.ru tls=on auth=on user= from= recipients= mailsize=7744 smtpstatus=250 smtpmsg='250 2.0.0 Ok: queued as 2DEA8A7F61' exitcode=EX_OK +Jul 28 19:04:01 host=mail.pvepro.ru tls=on auth=on user= from= recipients= mailsize=7697 smtpstatus=250 smtpmsg='250 2.0.0 Ok: queued as 5CE98A737C' exitcode=EX_OK +Jul 28 19:05:08 host=mail.pvepro.ru tls=on auth=on user= from= recipients= mailsize=7697 smtpstatus=250 smtpmsg='250 2.0.0 Ok: queued as E140AA737C' exitcode=EX_OK +Jul 29 08:48:58 host=mail.pvepro.ru tls=on auth=on user= from= recipients= mailsize=7669 smtpstatus=250 smtpmsg='250 2.0.0 Ok: queued as 227FBA7431' exitcode=EX_OK +Jul 29 20:33:47 host=mail.pvepro.ru tls=on auth=on user= from= recipients= mailsize=7685 smtpstatus=250 smtpmsg='250 2.0.0 Ok: queued as 9EBCFA7F0E' exitcode=EX_OK +Jul 30 16:48:00 host=mail.pvepro.ru tls=on auth=on user= from= recipients= mailsize=7687 smtpstatus=250 smtpmsg='250 2.0.0 Ok: queued as B1345A737C' exitcode=EX_OK +Aug 17 08:05:32 host=mail.pvepro.ru tls=on auth=on user= from= recipients= mailsize=7738 smtpstatus=250 smtpmsg='250 2.0.0 Ok: queued as 3EF03A7F8D' exitcode=EX_OK +Aug 17 08:07:52 host=mail.pvepro.ru tls=on auth=on user= from= recipients= mailsize=7738 smtpstatus=250 smtpmsg='250 2.0.0 Ok: queued as 6635BA73AA' exitcode=EX_OK +Aug 18 16:58:38 host=postbox.cloud.yandex.net tls=on auth=on user=ajeups2v3l2qfl9t1k9a from= recipients= mailsize=440 smtpstatus=250 smtpmsg='250 2.6.0 OK: queued as DKS89L8IPX05.3NYOOU5KRIBAX@ingress2-vla' exitcode=EX_OK +Aug 18 18:59:07 host=postbox.cloud.yandex.net tls=on auth=on user=ajeups2v3l2qfl9t1k9a from= recipients= mailsize=492 smtpstatus=250 smtpmsg='250 2.6.0 OK: queued as DKSATUF4C1XX.329CV6B95FH4R@ingress2-vla' exitcode=EX_OK +Aug 21 15:02:48 host=postbox.cloud.yandex.net tls=on auth=on user=ajeups2v3l2qfl9t1k9a from= recipients= mailsize=7672 smtpstatus=250 smtpmsg='250 2.6.0 OK: queued as DKUPOJOWNNPA.3CH5IJH2A8PW1@ingress2-klg' exitcode=EX_OK +Sep 02 19:39:11 host=postbox.cloud.yandex.net tls=on auth=on user=ajeups2v3l2qfl9t1k9a from= recipients= smtpstatus=550 smtpmsg='550 5.4.1 identity not found RequestID: 488b75b4-c264-42eb-b6fc-0cae6b175a7a' errormsg='envelope from address not accepted by the server' exitcode=EX_DATAERR +Sep 02 19:43:53 host=smtp.yandex.ru tls=on auth=on user= from= recipients= mailsize=257 smtpstatus=250 smtpmsg='250 2.0.0 Ok: queued on mail-nwsmtp-smtp-production-main-95.klg.yp-c.yandex.net 1788378233-qhqjKj2mO8c0-EjXDwBNJ' exitcode=EX_OK +Sep 02 19:44:53 host=smtp.yandex.ru tls=on auth=on user= from= recipients= mailsize=254 smtpstatus=250 smtpmsg='250 2.0.0 Ok: queued on mail-nwsmtp-smtp-production-main-71.sas.yp-c.yandex.net 1788378293-riqme1Oc6uQ0-JA3QSjr3' exitcode=EX_OK +Sep 02 21:33:45 host=smtp.yandex.ru tls=on auth=on user= from= recipients= mailsize=7477 smtpstatus=250 smtpmsg='250 2.0.0 Ok: queued on mail-nwsmtp-smtp-production-main-99.klg.yp-c.yandex.net 1788384825-iXsB0HPeHeA0-t67v0c58' exitcode=EX_OK +Sep 03 04:35:14 host=smtp.yandex.ru tls=on auth=on user= from= recipients= mailsize=2390 smtpstatus=250 smtpmsg='250 2.0.0 Ok: queued on mail-nwsmtp-smtp-production-main-85.iva.yp-c.yandex.net 1788410114-DZclo65g6Sw0-ym39kyd6' exitcode=EX_OK +Sep 03 04:44:35 host=smtp.yandex.ru tls=on auth=on user= from= recipients= mailsize=2412 smtpstatus=250 smtpmsg='250 2.0.0 Ok: queued on mail-nwsmtp-smtp-production-main-89.sas.yp-c.yandex.net 1788410675-ZicUTjSgH4Y0-ino0SiK0' exitcode=EX_OK +Sep 04 04:09:17 host=smtp.yandex.ru tls=on auth=on user= from= recipients= mailsize=7474 smtpstatus=250 smtpmsg='250 2.0.0 Ok: queued on mail-nwsmtp-smtp-production-main-88.klg.yp-c.yandex.net 1788494957-H9Anioed9mI0-YAPrVYsC' exitcode=EX_OK +Sep 04 04:09:44 host=smtp.yandex.ru tls=on auth=on user= from= recipients= mailsize=20404 smtpstatus=250 smtpmsg='250 2.0.0 Ok: queued on mail-nwsmtp-smtp-production-main-59.klg.yp-c.yandex.net 1788494984-h9AZQbdd08c0-OXrP5vgQ' exitcode=EX_OK +Sep 04 04:19:31 host=smtp.yandex.ru tls=on auth=on user= from= recipients= mailsize=7632 smtpstatus=250 smtpmsg='250 2.0.0 Ok: queued on mail-nwsmtp-smtp-production-main-98.sas.yp-c.yandex.net 1788495571-UJAjYwgdJCg0-xBGi7vHN' exitcode=EX_OK +Sep 04 06:00:34 host=smtp.yandex.ru tls=on auth=on user= from= recipients= mailsize=7703 smtpstatus=250 smtpmsg='250 2.0.0 Ok: queued on mail-nwsmtp-smtp-production-main-59.iva.yp-c.yandex.net 1788501634-X0CkCDGeJqM0-ewqs8N37' exitcode=EX_OK +Sep 08 08:25:56 host=smtp.yandex.ru tls=on auth=on user= from= recipients= mailsize=7704 smtpstatus=250 smtpmsg='250 2.0.0 Ok: queued on mail-nwsmtp-smtp-production-main-89.sas.yp-c.yandex.net 1788855956-tPIpFURgCSw0-0bD0dt06' exitcode=EX_OK +Sep 10 00:10:02 host=smtp.yandex.ru tls=on auth=on user= from= recipients= smtpstatus=553 smtpmsg='553 5.7.1 Sender address rejected: user not found. 1788999002-1ACLU4egAmI0-BvLQE2bE' errormsg='envelope from address not accepted by the server' exitcode=EX_DATAERR +Sep 10 00:10:03 host=smtp.yandex.ru tls=on auth=on user= from= recipients= mailsize=10460 smtpstatus=250 smtpmsg='250 2.0.0 Ok: queued on mail-nwsmtp-smtp-production-main-71.sas.yp-c.yandex.net 1788999003-3ACiEWkcAmI0-sgTWLJGA' exitcode=EX_OK +Sep 10 00:10:05 host=smtp.yandex.ru tls=on auth=on user= from= recipients= mailsize=10599 smtpstatus=250 smtpmsg='250 2.0.0 Ok: queued on mail-nwsmtp-smtp-production-main-60.sas.yp-c.yandex.net 1788999005-4ACnvojfQa60-IhiFG02U' exitcode=EX_OK +Sep 10 00:10:06 host=smtp.yandex.ru tls=on auth=on user= from= recipients= smtpstatus=553 smtpmsg='553 5.7.1 Sender address rejected: user not found. 1788999006-6ACgTJhfxuQ0-ykq2gT2m' errormsg='envelope from address not accepted by the server' exitcode=EX_DATAERR +Sep 10 00:10:07 host=smtp.yandex.ru tls=on auth=on user= from= recipients= smtpstatus=553 smtpmsg='553 5.7.1 Sender address rejected: user not found. 1788999007-7ACYnudg6mI0-wrzZ9ta2' errormsg='envelope from address not accepted by the server' exitcode=EX_DATAERR +Sep 10 00:10:08 host=smtp.yandex.ru tls=on auth=on user= from= recipients= mailsize=10224 smtpstatus=250 smtpmsg='250 2.0.0 Ok: queued on mail-nwsmtp-smtp-production-main-58.iva.yp-c.yandex.net 1788999008-8ACm4u3oe8c0-EAVVXjUV' exitcode=EX_OK +Sep 13 16:14:26 host=smtp.yandex.ru tls=on auth=on user= from= recipients= mailsize=7702 smtpstatus=250 smtpmsg='250 2.0.0 Ok: queued on mail-nwsmtp-smtp-production-main-67.vla.yp-c.yandex.net 1789316066-QEVwdXHe9Os0-PH6Ad0od' exitcode=EX_OK +Sep 19 00:10:02 host=smtp.yandex.ru tls=on auth=on user= from= recipients= smtpstatus=553 smtpmsg='553 5.7.1 Sender address rejected: user not found. 1789776602-2ALqagFfpGk0-vq5tlHNF' errormsg='envelope from address not accepted by the server' exitcode=EX_DATAERR +Sep 19 00:10:03 host=smtp.yandex.ru tls=on auth=on user= from= recipients= smtpstatus=553 smtpmsg='553 5.7.1 Sender address rejected: user not found. 1789776603-3ALZ9cFg74Y0-Rbt8yJgu' errormsg='envelope from address not accepted by the server' exitcode=EX_DATAERR +Sep 19 00:10:04 host=smtp.yandex.ru tls=on auth=on user= from= recipients= smtpstatus=553 smtpmsg='553 5.7.1 Sender address rejected: user not found. 1789776604-4AL5ABFdSSw0-XwXfZVbq' errormsg='envelope from address not accepted by the server' exitcode=EX_DATAERR +Sep 19 00:10:05 host=smtp.yandex.ru tls=on auth=on user= from= recipients= smtpstatus=553 smtpmsg='553 5.7.1 Sender address rejected: user not found. 1789776605-5ALSb5FdeKo0-ugaNiHmf' errormsg='envelope from address not accepted by the server' exitcode=EX_DATAERR +Sep 19 00:10:06 host=smtp.yandex.ru tls=on auth=on user= from= recipients= smtpstatus=553 smtpmsg='553 5.7.1 Sender address rejected: user not found. 1789776606-6ALAYdFdNuQ0-dL2d7DHS' errormsg='envelope from address not accepted by the server' exitcode=EX_DATAERR +Sep 19 00:10:07 host=smtp.yandex.ru tls=on auth=on user= from= recipients= smtpstatus=553 smtpmsg='553 5.7.1 Sender address rejected: user not found. 1789776607-7ALu7bFe4Gk0-WcjF4jNs' errormsg='envelope from address not accepted by the server' exitcode=EX_DATAERR +=== JOB RUNNER === +JOBRUNNER_META=/root/scripts/forum-xenforo-run-jobs.sh mode=700 owner=root group=root size=2343 +JOBRUNNER_SHA256=5cc6c6cba317e0f0c4b1b13f375d8677b3c9c96d0416f998315a7d4e05e8a1aa PATH=/root/scripts/forum-xenforo-run-jobs.sh +JOBRUNNER_META=/etc/cron.d/forum-xenforo-run-jobs mode=644 owner=root group=root size=183 +JOBRUNNER_SHA256=20cf9e4d059148c14094d145e778bb307b21183f608ff9b430dd620b9ece75c4 PATH=/etc/cron.d/forum-xenforo-run-jobs +=== XENFORO SITES === +XENFORO_ROOT_COUNT=6 +SITE_BEGIN=1 +ROOT_SHA256=90d10058f0574d8a5ff1ff412fb7b6a1eab2119d09c8b8162a0f8ca55d15180f +DB_CONNECT=OK +BOARD_DOMAIN=codevipe.ru +EMAIL_TRANSPORT={"emailTransport":"sendmail"} +DEFAULTEMAILADDRESS_PRESENT=true +DEFAULTEMAILADDRESS_SHA256=2d4e7d3cf84ec1ba079400cf62ca99e7464a221a686ebe73405b73fb01ffe6f4 +BOUNCEEMAILADDRESS_PRESENT=true +BOUNCEEMAILADDRESS_SHA256=1b5fc8a44aa74c2f02b7ce893f8a0e0a901d60b3f5e68b39c9448e7265117265 +CONTACTEMAILADDRESS_PRESENT=true +CONTACTEMAILADDRESS_SHA256=0d4cf60ca8cd53a12ad50dc5419d3c9c32192f0852000eb10051fbdef52b04bf +XF_ERROR_COUNT_24H=0 +XF_ERROR_COUNT_7D=2 +XF_ERROR_COUNT_30D=2 +XF_ERROR_COUNT_ALL=2 +XF_ERROR_TYPE=XF\Db\InvalidQueryException COUNT=1 LAST=1789960536 +XF_ERROR_TYPE=Symfony\Component\Mailer\Exception\TransportException COUNT=1 LAST=1789776602 +XF_ERROR_ROW ID=2 TS=1789960536 TYPE=XF\Db\InvalidQueryException FILE=AbstractStatement.php LINE=225 MSG=MySQL statement prepare error [1064]: You have an error in your SQL syntax; check the manual that corresponds to your MariaDB server version for the right syntax to use near 'FROM xf_template_modification WHERE addon_id='Gram/Support' ORDER MSG_SHA256=9ad3a33fd2ebdd213c22e2156d0358b02f9f6f82e2027f3101055bbfe5dd83b5 +XF_ERROR_ROW ID=1 TS=1789776602 TYPE=Symfony\Component\Mailer\Exception\TransportException FILE=ProcessStream.php LINE=74 MSG=Email to from failed: Process failed with exit code 65: sendmail: envelope from address not accepted by the server sendmail: server message: 553 5.7.1 Sender address rejected: user not found. 1789776602-2ALqagFfpGk0- MSG_SHA256=6ec17ccf28a8f55861f6ceb88ab722165bd922e90a12976f2755bc9db10c3638 +XF_MAIL_QUEUE_TABLE=MISSING +XF_JOB_COUNT=2 +SITE_PHP_AUDIT_RC=0 +SITE_END=1 +SITE_BEGIN=2 +ROOT_SHA256=d33b4c353c031284c61f9632f49971cd1f833ab4f48c0da27e4fdf81c373652e +DB_CONNECT=OK +BOARD_DOMAIN=dsmods.ru +EMAIL_TRANSPORT={"emailTransport":"sendmail"} +DEFAULTEMAILADDRESS_PRESENT=true +DEFAULTEMAILADDRESS_SHA256=953017f372fc160381792c8f72822095c821e5f0c4519776c44cfa01fb6c16f1 +BOUNCEEMAILADDRESS_PRESENT=true +BOUNCEEMAILADDRESS_SHA256=ba70f629ed27c307125a0538ca1c95049ac153aea2b21a474afeb4e3ad60d08d +CONTACTEMAILADDRESS_PRESENT=true +CONTACTEMAILADDRESS_SHA256=0d4cf60ca8cd53a12ad50dc5419d3c9c32192f0852000eb10051fbdef52b04bf +XF_ERROR_COUNT_24H=0 +XF_ERROR_COUNT_7D=2 +XF_ERROR_COUNT_30D=2 +XF_ERROR_COUNT_ALL=2 +XF_ERROR_TYPE=XF\Db\InvalidQueryException COUNT=1 LAST=1789960537 +XF_ERROR_TYPE=Symfony\Component\Mailer\Exception\TransportException COUNT=1 LAST=1789776606 +XF_ERROR_ROW ID=2 TS=1789960537 TYPE=XF\Db\InvalidQueryException FILE=AbstractStatement.php LINE=225 MSG=MySQL statement prepare error [1064]: You have an error in your SQL syntax; check the manual that corresponds to your MariaDB server version for the right syntax to use near 'FROM xf_template_modification WHERE addon_id='Gram/Support' ORDER MSG_SHA256=9ad3a33fd2ebdd213c22e2156d0358b02f9f6f82e2027f3101055bbfe5dd83b5 +XF_ERROR_ROW ID=1 TS=1789776606 TYPE=Symfony\Component\Mailer\Exception\TransportException FILE=ProcessStream.php LINE=74 MSG=Email to from failed: Process failed with exit code 65: sendmail: envelope from address not accepted by the server sendmail: server message: 553 5.7.1 Sender address rejected: user not found. 1789776606-6ALAYdFdNuQ0- MSG_SHA256=59e1133715aedc5b530f2a2f87f31d659af13424d86fdc7b06d1331635178a0c +XF_MAIL_QUEUE_TABLE=MISSING +XF_JOB_COUNT=2 +SITE_PHP_AUDIT_RC=0 +SITE_END=2 +SITE_BEGIN=3 +ROOT_SHA256=e06523a33d0b38a0ec6532783b5e63542eeba1303e68f2efad63214fd1e8908f +DB_CONNECT=OK +BOARD_DOMAIN=gamevipe.ru +EMAIL_TRANSPORT={"emailTransport":"sendmail"} +DEFAULTEMAILADDRESS_PRESENT=true +DEFAULTEMAILADDRESS_SHA256=ded102c05ab32e41f2ada4d15b63c6adb848e14e76154015f294b751459a75dc +BOUNCEEMAILADDRESS_PRESENT=true +BOUNCEEMAILADDRESS_SHA256=3c6809f1cc3dd12044ad384b8d944a408bd84b4320af0bb505a5fb5ebf3f10e3 +CONTACTEMAILADDRESS_PRESENT=true +CONTACTEMAILADDRESS_SHA256=0d4cf60ca8cd53a12ad50dc5419d3c9c32192f0852000eb10051fbdef52b04bf +XF_ERROR_COUNT_24H=0 +XF_ERROR_COUNT_7D=1 +XF_ERROR_COUNT_30D=1 +XF_ERROR_COUNT_ALL=1 +XF_ERROR_TYPE=Symfony\Component\Mailer\Exception\TransportException COUNT=1 LAST=1789776603 +XF_ERROR_ROW ID=1 TS=1789776603 TYPE=Symfony\Component\Mailer\Exception\TransportException FILE=ProcessStream.php LINE=74 MSG=Email to from failed: Process failed with exit code 65: sendmail: envelope from address not accepted by the server sendmail: server message: 553 5.7.1 Sender address rejected: user not found. 1789776603-3ALZ9cFg74Y0- MSG_SHA256=f0aeb49dcba2c41db1f69663611a12e57913ca0b631287cdcc3c3b2a8db73f51 +XF_MAIL_QUEUE_TABLE=MISSING +XF_JOB_COUNT=3 +SITE_PHP_AUDIT_RC=0 +SITE_END=3 +SITE_BEGIN=4 +ROOT_SHA256=127c45667b0b6b6f23d20d6cc66d094b61a854801e42025ad2cb81ab5bfbc03e +DB_CONNECT=OK +BOARD_DOMAIN=hkmods.ru +EMAIL_TRANSPORT={"emailTransport":"sendmail"} +DEFAULTEMAILADDRESS_PRESENT=true +DEFAULTEMAILADDRESS_SHA256=ef08c646d1d1fc56ef350fba44e29f2117996462bf46d9d14fa33c0688f9f4e3 +BOUNCEEMAILADDRESS_PRESENT=true +BOUNCEEMAILADDRESS_SHA256=0097bdfb68ae6258a87c12c8dae4823421d4209086a2f190cb189defbda96bc3 +CONTACTEMAILADDRESS_PRESENT=true +CONTACTEMAILADDRESS_SHA256=0d4cf60ca8cd53a12ad50dc5419d3c9c32192f0852000eb10051fbdef52b04bf +XF_ERROR_COUNT_24H=0 +XF_ERROR_COUNT_7D=1 +XF_ERROR_COUNT_30D=1 +XF_ERROR_COUNT_ALL=1 +XF_ERROR_TYPE=Symfony\Component\Mailer\Exception\TransportException COUNT=1 LAST=1789776604 +XF_ERROR_ROW ID=1 TS=1789776604 TYPE=Symfony\Component\Mailer\Exception\TransportException FILE=ProcessStream.php LINE=74 MSG=Email to from failed: Process failed with exit code 65: sendmail: envelope from address not accepted by the server sendmail: server message: 553 5.7.1 Sender address rejected: user not found. 1789776604-4AL5ABFdSSw0- MSG_SHA256=03dcb6ecd92e29dde8dcccc9776bb4af1985d7d525bfae5cf9b5bb4db579de95 +XF_MAIL_QUEUE_TABLE=MISSING +XF_JOB_COUNT=3 +SITE_PHP_AUDIT_RC=0 +SITE_END=4 +SITE_BEGIN=5 +ROOT_SHA256=d249590aad8f87daf95d1fed623405edb307aa6c9b8123b2aaf89da88fc117ab +DB_CONNECT=OK +BOARD_DOMAIN=kingofwolk.ru +EMAIL_TRANSPORT={"emailTransport":"sendmail"} +DEFAULTEMAILADDRESS_PRESENT=true +DEFAULTEMAILADDRESS_SHA256=e3b5d95ee979b3970d8ac7a07e503a6f5247963b701b1d1ad0b4aca12fc41d3a +BOUNCEEMAILADDRESS_PRESENT=true +BOUNCEEMAILADDRESS_SHA256=e49d0b3b9f084cf1223dd2b108019277e2b51caf2cf1a63b3ba3853765aefc65 +CONTACTEMAILADDRESS_PRESENT=true +CONTACTEMAILADDRESS_SHA256=0d4cf60ca8cd53a12ad50dc5419d3c9c32192f0852000eb10051fbdef52b04bf +XF_ERROR_COUNT_24H=0 +XF_ERROR_COUNT_7D=1 +XF_ERROR_COUNT_30D=1 +XF_ERROR_COUNT_ALL=1 +XF_ERROR_TYPE=Symfony\Component\Mailer\Exception\TransportException COUNT=1 LAST=1789776607 +XF_ERROR_ROW ID=1 TS=1789776607 TYPE=Symfony\Component\Mailer\Exception\TransportException FILE=ProcessStream.php LINE=74 MSG=Email to from failed: Process failed with exit code 65: sendmail: envelope from address not accepted by the server sendmail: server message: 553 5.7.1 Sender address rejected: user not found. 1789776607-7ALu7bFe4Gk0- MSG_SHA256=075f2b1963657d083c11e214ecc8cfd4e01d30f9ac1723bc440808730e6fb809 +XF_MAIL_QUEUE_TABLE=MISSING +XF_JOB_COUNT=2 +SITE_PHP_AUDIT_RC=0 +SITE_END=5 +SITE_BEGIN=6 +ROOT_SHA256=b5bc9e40848a038adb66a5656917858172c1b70a14ccbe680e9d8f2260dda046 +DB_CONNECT=OK +BOARD_DOMAIN=zakrutim.ru +EMAIL_TRANSPORT={"emailTransport":"sendmail"} +DEFAULTEMAILADDRESS_PRESENT=true +DEFAULTEMAILADDRESS_SHA256=b719f473d6ff99139fef2749e636437b76aaf482963e82dde4ac69a296a2eab9 +BOUNCEEMAILADDRESS_PRESENT=true +BOUNCEEMAILADDRESS_SHA256=a93aa0f45435a0e2c159c501fbe3cc70deaff82401d29d87a972e633f91df83a +CONTACTEMAILADDRESS_PRESENT=true +CONTACTEMAILADDRESS_SHA256=0d4cf60ca8cd53a12ad50dc5419d3c9c32192f0852000eb10051fbdef52b04bf +XF_ERROR_COUNT_24H=0 +XF_ERROR_COUNT_7D=1 +XF_ERROR_COUNT_30D=1 +XF_ERROR_COUNT_ALL=1 +XF_ERROR_TYPE=Symfony\Component\Mailer\Exception\TransportException COUNT=1 LAST=1789776605 +XF_ERROR_ROW ID=1 TS=1789776605 TYPE=Symfony\Component\Mailer\Exception\TransportException FILE=ProcessStream.php LINE=74 MSG=Email to from failed: Process failed with exit code 65: sendmail: envelope from address not accepted by the server sendmail: server message: 553 5.7.1 Sender address rejected: user not found. 1789776605-5ALSb5FdeKo0- MSG_SHA256=894c6c5636182853711e897fbcf34d5881646135b6dffd7360737376e789d384 +XF_MAIL_QUEUE_TABLE=MISSING +XF_JOB_COUNT=2 +SITE_PHP_AUDIT_RC=0 +SITE_END=6 +=== RECENT SERVICE ERROR COUNTS === +JOURNAL_ERROR_COUNT_24H_nginx=0 +JOURNAL_ERROR_COUNT_24H_mariadb=0 +JOURNAL_ERROR_COUNT_24H_mysql=0 +JOURNAL_ERROR_COUNT_24H_php8_3_fpm=0 +VM160_GUEST_AUDIT_END=true +VM160_GUEST_STDERR_PRESENT=false +VM160_QGA_PARSE_RC=0 +PVE02_REMOTE_END=true +FAILURE=NONE +DECISION=PASS_PVE02_VM160_AUDIT +NEXT_GATE=REVIEW_FOR_BOUNDED_REPAIR +TASK_RESULT=READ_ONLY_PVE02_VM160_FORUM_AUDIT_COMPLETE +HOMELAB_RESULT_CONTRACT={"version":1,"command_id":"FORUM-260923-RCA-PVE02-VM160-AUDITC7A1R1","status":"OK","changes_made":false,"rollback_started":false,"rollback_restored":null} + +OUTPUT_END +CHAT_OUTPUT_END diff --git a/runtime/latest.json b/runtime/latest.json index 762eb476..92c90162 100644 --- a/runtime/latest.json +++ b/runtime/latest.json @@ -1,19 +1,19 @@ { "schema_version": 1, "channel": "homelab-runtime", - "command_id": "SUPPORT-260923-HOMELAB-MOLDOVA-RETIRE-PREFLIGHT-1130R1", + "command_id": "FORUM-260923-RCA-PVE02-VM160-AUDITC7A1R1", "status": "OK", "rc": 0, "host": "pve01", "mode": "read-only", - "component": "homelab-moldova-retire-preflight", - "started_at_utc": "2026-09-23T20:58:51Z", - "finished_at_utc": "2026-09-23T20:58:56Z", + "component": "forum-pve02-vm160-audit", + "started_at_utc": "2026-09-23T21:02:32Z", + "finished_at_utc": "2026-09-23T21:02:38Z", "reference_register_checked": true, "reference_sha256": "5a3d8e5154c41cb582a4e0aca68090be1f0138918bf82131a948df326f9d8d66", "error_register_checked": true, "error_register_sha256": "3b09a553ec0f527ed3afeed4753f52a74ee3036045bbfb3c685e9f8af4ba7ba0", - "command_sha256": "724447a90ca704066487f2c058f6cbf14e3086e7b399e385e5bca6df62d1cd70", + "command_sha256": "d0d58f5b0d26ba9465d13d1ce6a174370a5532296fef88f757fd4fb1f2152a12", "duplicate_failed_command_blocked": false, "block_reason": null, "execution_started": true, @@ -30,9 +30,9 @@ "secrets_included": false, "private_addresses_included": false, "raw_evidence_retained_locally": true, - "raw_evidence_sha256": "f75ed52bc7c1eb7c5418acf9a6f3c7d372de8ddb2bf445b6eb507958c8172cf8", - "sanitized_output_sha256": "c0f54cefd55350849427898c5de7f25cde594a78aba7943a1b3599863593f6ac", + "raw_evidence_sha256": "73f06c6c0cffab9eaddcd407da9d716944ca4ef348dd03eacfc4e7c8ceea1ea3", + "sanitized_output_sha256": "73f06c6c0cffab9eaddcd407da9d716944ca4ef348dd03eacfc4e7c8ceea1ea3", "output_truncated_in_json": false, "full_sanitized_output_url": "https://git.gram1.ru/.well-known/homelab-runtime/latest.txt", - "output": "WORKERS2_MOLDOVA_RETIRE_PREFLIGHT_BEGIN=1\nCOMMAND_ID=SUPPORT-260923-HOMELAB-MOLDOVA-RETIRE-PREFLIGHT-1130R1\nMODE=read-only\nCOMPONENT=homelab-moldova-retire-preflight\nMUTATION_BOUNDARY=NONE;LOCAL_DEPENDENCY_DISCOVERY_ONLY;NO_PROVIDER_CALL;NO_TARGET_SSH;NO_NETWORK_MUTATION;NO_SYSTEMD_ACTION;NO_DNS_CHANGE;NO_BACKUP_MUTATION;NO_GIT_WRITE;NO_SECRET_VALUE_READOUT\nPREFLIGHT_PURPOSE=PROVE_WHAT_MUST_BE_DETACHED_OR_REPLACED_BEFORE_MOLDOVA_VPS_CAN_BE_QUARANTINED_OR_DELETED\nREFERENCE_SHA256=5a3d8e5154c41cb582a4e0aca68090be1f0138918bf82131a948df326f9d8d66\nERROR_REGISTER_SHA256=3b09a553ec0f527ed3afeed4753f52a74ee3036045bbfb3c685e9f8af4ba7ba0\nRUNNER_SHA256=b248a4c32c9cc64e5747e7dce6c7fc0a23f5124a77c71ce72e27a81aceae9d2d\nAUTHORITY_GATE=PASS\nFRESH_CONTEXT_BEGIN=1\nNOW_UTC=2026-09-23T20:58:51Z\nHOSTNAME=pve01\nFRESH_CONTEXT_END=1\nHANDOFF_CANDIDATES_BEGIN=1\nHANDOFF_CANDIDATE_COUNT=4\nHANDOFF_FILE=/srv/homelab-ops/kb/current/04_P0_NEXT.md\nHANDOFF_MTIME=2026-08-21 00:06:53.672037607 +0300\nHANDOFF_SHA256=0f6eb8b23382becc1868e8a22318d5b7629184568205bad47c3c38967a346b8f\n5:3. Canonical post-migration CR including NetBird new USA, Postbox/Null MX, Moldova V3/Gotify, old-USA retirement window and dependencies.\n7:5. VM160 backup-state audit.\n12:Do not mix unrelated changes into these gates (e.g. NetBird version upgrade/hostname change during SSH hardening).\nHANDOFF_FILE=/srv/homelab-ops/kb/changes/CR-2026-0087-HANDOFF.md\nHANDOFF_MTIME=2026-08-21 00:06:53.671922428 +0300\nHANDOFF_SHA256=a8d91b33b36e462bf3bb48defd12e121ed06edcb12d2dab3a9be50b2c5cdcb4d\n13:- Scrutiny and crypto chart-health runtime are removed.\nHANDOFF_FILE=/srv/homelab-ops/kb/changes/CR-2026-0086-HANDOFF.md\nHANDOFF_MTIME=2026-08-21 00:06:53.671835127 +0300\nHANDOFF_SHA256=faab79cda5c6b31162b68812fc468359c0b9b94892c8f1e3b31c626d6704264a\n26:3. load regression/error registry;\n68:Failure: clean clone had no Git `user.name` / `user.email`.\n138:V4 service was deployed side-by-side and later verified healthy, but public\n199:- regression registry\nHANDOFF_FILE=/srv/homelab-ops/kb/AI_CONTEXT.md\nHANDOFF_MTIME=2026-08-21 00:06:53.671629955 +0300\nHANDOFF_SHA256=5fc43e34a1f0714ddacf7c00ed5c4bbaf788df3b8407dcfebd1333a23a623acc\n15:- pve01 `[IP]`, operator historical NetBird path `root@[IP]`, MobaXterm key label `eddsa-key-20260617`.\n26:## Mail/forums\n30:## NetBird migration\n32:Production NetBird is new USA `[IP]`, Debian 12, hostname `nb2`, NetBird `0.73.2`. Exact deployment caveat: `/api/health` = 404 and combined `:9000/health` = 503; these are not generic readiness gates for this deployment. Old USA `[IP]` has Mailcow/NetBird server stopped and observer disabled, but remains in rollback window. Permanent new-USA operator SSH hardening is still P0.\n34:## Moldova\n38:## Backups/cloud\n40:Canonical production backup remotes remain Mail A/B. Full SITE-DR is SEALED; do not rerun merely because architecture changed. Cloud.ru bucket `homelab-dr-backup`: versioning **ON** (this supersedes older OFF docs), writer/reader OFF, no real workload. Yandex Object Storage bucket not created; latest verified billing threshold is `-600 RUB`, target 0, support request pending. Bucket creation remains blocked until live 0 proof.\nHANDOFF_CANDIDATES_END=1\nREFERENCE_MOLDOVA_SNIPPETS_BEGIN=1\n4:SUPERSEDES_OLDER_TOPOLOGY_AND_BACKUP_SNAPSHOTS=YES\n23:- VM180 cluster-admin | DESTROYED | backup registry lifecycle=RETIRED_PRESERVE\n24:- CT200 skladchik-mod | DESTROYED | backup registry lifecycle=RETIRED_PRESERVE\n25:- Scrutiny | RETIRED | active service/Podman removed | backup registry lifecycle=RETIRED_PRESERVE | historical recovery evidence preserved\n28:### BACKUP_CURRENT\n29:BACKUP_STATUS=GREEN\n30:BACKUP_REGISTRY_TOTAL=21\n31:BACKUP_ACTIVE_VM=11\n32:BACKUP_RETIRED_VM=2\n33:BACKUP_ACTIVE_APP=4\n34:BACKUP_RETIRED_APP=3\n35:BACKUP_ACTIVE_CONTROL=1\n36:BACKUP_FRESH_TARGETS=15\n37:BACKUP_SOURCE_DRIFT=0\n38:BACKUP_RESUME=0\n39:BACKUP_PAYLOAD_LEAKS=0\n40:BACKUP_VM_QUEUE=inactive/success\n41:BACKUP_APP_QUEUE=inactive/success\n55:- Immich machine-learning runtime recovered; current stock healthcheck passes and server-to-ML /ping returns HTTP 200.\n72:HEALTH_WARN_ERROR_COUNT=14\n74:NOTE forum-prod VMID160 is not in homelab-nightly-all; it has separate local/cloud backup chain\n75:NOTE edge-vm disk scsi1 backup=0 risk must be documented\n77:NOTE edge health WARN/ERROR items should be documented as known current states\n79:HEALTH_NOT_OK\n81:HEALTH_NOT_OK item=backup-restore-coverage-slo.txt line=STATUS=WARN\n82:HEALTH_NOT_OK item=backup-restore-dashboard.txt line=STATUS=WARN TS=2026-06-29T18:10:09Z ITEMS=80 NOT_OK=13\n83:HEALTH_NOT_OK item=container-image-lifecycle-slo.txt line=STATUS=WARN\n84:HEALTH_NOT_OK item=cron-job-monitoring-slo.txt line=STATUS=WARN\n85:HEALTH_NOT_OK item=direct-heartbeat-pilot-readiness.txt line=STATUS=WARN\n86:HEALTH_NOT_OK item=healthchecks-heartbeat-coverage.txt line=STATUS=WARN\n87:HEALTH_NOT_OK item=healthchecks-job-coverage.txt line=STATUS=WARN\n88:HEALTH_NOT_OK item=healthchecks-job-monitoring-slo.txt line=STATUS=WARN\n89:HEALTH_NOT_OK item=homepage-service-catalog-slo.txt line=STATUS=WARN\n90:HEALTH_NOT_OK item=ingress-dns-route-slo.txt line=STATUS=WARN\n91:HEALTH_NOT_OK item=job-monitoring-slo.txt line=STATUS=WARN\n92:HEALTH_NOT_OK item=observability-health-surface-slo.txt line=STATUS=WARN\nREFERENCE_MOLDOVA_SNIPPETS_END=1\nSYSTEMD_STATE_BEGIN=1\nUNIT|name=pvepro-relay-healthcheck.timer|load=not-found|active=inactive|sub=dead|result=success|enabled=not-found\nUNIT|name=pvepro-relay-healthcheck.service|load=not-found|active=inactive|sub=dead|result=success|enabled=not-found\nUNIT|name=homelab-backup-v2.timer|load=loaded|active=active|sub=waiting|result=success|enabled=enabled\nUNIT|name=homelab-backup-v2.service|load=loaded|active=inactive|sub=dead|result=success|enabled=static\nUNIT|name=homelab-backup-audit.timer|load=loaded|active=active|sub=waiting|result=success|enabled=enabled\nUNIT|name=homelab-backup-audit.service|load=loaded|active=failed|sub=failed|result=exit-code|enabled=static\nUNIT|name=homelab-alertmanager-backup-health.timer|load=loaded|active=active|sub=waiting|result=success|enabled=enabled\nUNIT|name=homelab-alertmanager-backup-health.service|load=loaded|active=inactive|sub=dead|result=success|enabled=static\nUNIT|name=netbird-peers-health.timer|load=loaded|active=active|sub=waiting|result=success|enabled=enabled\nUNIT|name=netbird-peers-health.service|load=loaded|active=failed|sub=failed|result=exit-code|enabled=static\nUNIT|name=netbird.service|load=loaded|active=active|sub=running|result=success|enabled=enabled\nUNIT|name=netbird-vps-backup.timer|load=not-found|active=inactive|sub=dead|result=success|enabled=not-found\nUNIT|name=netbird-vps-backup.service|load=not-found|active=inactive|sub=dead|result=success|enabled=not-found\nUNIT|name=netbird-vps-offhost-sync.timer|load=not-found|active=inactive|sub=dead|result=success|enabled=not-found\nUNIT|name=netbird-vps-offhost-sync.service|load=not-found|active=inactive|sub=dead|result=success|enabled=not-found\nUNIT|name=homelab-crowdsec-capi-netbird-egress.service|load=not-found|active=inactive|sub=dead|result=success|enabled=not-found\nUNIT|name=privoxy.service|load=not-found|active=inactive|sub=dead|result=success|enabled=not-found\nSYSTEMD_STATE_END=1\nBACKUP_AUDIT_FAILURE_EVIDENCE_BEGIN=1\nSep 23 [IPV6] pve01 homelab-backup-audit[2268991]: TARGET|logical_id=app-filebrowser|kind=APP|state=FRESH|decision=SKIP_FRESH|legacy_age_seconds=14746|generation=app-filebrowser-pve01-20260923T143005Z\nSep 23 [IPV6] pve01 homelab-backup-audit[2268991]: TARGET|logical_id=app-hbarhub|kind=APP|state=FRESH|decision=SKIP_FRESH|legacy_age_seconds=14574|generation=app-hbarhub-pve01-20260923T143158Z\nSep 23 [IPV6] pve01 homelab-backup-audit[2268991]: TARGET|logical_id=app-immich|kind=APP|state=FRESH|decision=SKIP_FRESH|legacy_age_seconds=14674|generation=app-immich-pve01-20260923T143041Z\nSep 23 [IPV6] pve01 homelab-backup-audit[2268991]: TARGET|logical_id=app-skladchik-reports-monitor|kind=APP|state=FRESH|decision=SKIP_FRESH|legacy_age_seconds=14724|generation=app-skladchik-reports-monitor-pve01-20260923T143025Z\nSep 23 [IPV6] pve01 homelab-backup-audit[2268991]: TARGET|logical_id=control-bootstrap-v2|kind=CONTROL|state=FRESH|age_seconds=36576|generation=control-bootstrap-v2-pve01-20260916T120057Z\nSep 23 [IPV6] pve01 homelab-backup-audit[2268991]: TARGET|logical_id=vm110|kind=VM|state=FRESH|source=V2|v2_age_seconds=20203|legacy_age_seconds=1688189|replica_count=2|state_id=ct110|generation=v2-ct110-20260923T125546Z-34408ffe4182\nSep 23 [IPV6] pve01 homelab-backup-audit[2268991]: TARGET|logical_id=vm111|kind=VM|state=FRESH|source=V2|v2_age_seconds=18061|replica_count=2|state_id=ct111|generation=v2-ct111-20260923T133222Z-0d3f5fae9c35\nSep 23 [IPV6] pve01 homelab-backup-audit[2268991]: TARGET|logical_id=vm112|kind=VM|state=FRESH|source=V2|v2_age_seconds=71147|legacy_age_seconds=1726684|replica_count=2|state_id=ct112|generation=v2-ct112-20260922T224902Z-8457b570052f\nSep 23 [IPV6] pve01 homelab-backup-audit[2268991]: TARGET|logical_id=vm113|kind=VM|state=FRESH|source=V2|v2_age_seconds=69278|replica_count=2|state_id=ct113|generation=v2-ct113-20260922T231959Z-9fe7d084a330\nSep 23 [IPV6] pve01 homelab-backup-audit[2268991]: TARGET|logical_id=vm130|kind=VM|state=FRESH|source=V2|v2_age_seconds=61252|replica_count=2|state_id=vm130|generation=v2-vm130-20260923T011558Z-c9498d37d3d7\nSep 23 [IPV6] pve01 homelab-backup-audit[2268991]: TARGET|logical_id=vm150|kind=VM|state=FRESH|source=V2|v2_age_seconds=66934|legacy_age_seconds=1688305|replica_count=2|state_id=vm150|generation=v2-vm150-20260922T235842Z-71b8fcec59c9\nSep 23 [IPV6] pve01 homelab-backup-audit[2268991]: TARGET|logical_id=vm160|kind=VM|state=FRESH|source=V2|v2_age_seconds=60152|replica_count=2|state_id=vm160|generation=v2-vm160-20260923T014608Z-f76d7671af5a\nSep 23 [IPV6] pve01 homelab-backup-audit[2268991]: TARGET|logical_id=vm170|kind=VM|state=FRESH|source=V2|v2_age_seconds=51801|legacy_age_seconds=632926|replica_count=2|state_id=vm170|generation=v2-vm170-20260923T035958Z-923adacbc94b\nSep 23 [IPV6] pve01 homelab-backup-audit[2268991]: TARGET|logical_id=vm171|kind=VM|state=FRESH|source=V2|v2_age_seconds=58355|legacy_age_seconds=662458|replica_count=2|state_id=vm171|generation=v2-vm171-20260923T021946Z-4d6e4f578075\nSep 23 [IPV6] pve01 homelab-backup-audit[2268991]: TARGET|logical_id=vm190|kind=VM|state=FRESH|source=V2|v2_age_seconds=54272|replica_count=2|state_id=vm190|generation=v2-vm190-20260923T032759Z-314d9ba2ec35\nSep 23 [IPV6] pve01 homelab-backup-audit[2268991]: TARGET|logical_id=vm9130|kind=VM|state=FRESH|source=V2|v2_age_seconds=39367|replica_count=2|state_id=vm9130|generation=v2-vm9130-20260923T071932Z-c0492db6535b\nSep 23 [IPV6] pve01 homelab-backup-audit[2268991]: ISSUE|severity=HIGH|code=LOCK_BUSY|logical_id=/run/homelab-backup-heavy.lock|root_cause=ACTIVE_OR_ORPHAN_LOCK\nSep 23 [IPV6] pve01 homelab-backup-audit[2268991]: SUMMARY|AUDIT_STATUS=OK|BACKUP_STATUS=RED|ACTIVE_COUNT=16|ISSUE_COUNT=1|CLOUD_A=OK|CLOUD_B=OK|CLOUD_B_DRILL=OK|DR_PROVEN=YES|SOURCE_OFFHOST=OK\nSep 23 [IPV6] pve01 systemd[1]: homelab-backup-audit.service: Main process exited, code=exited, status=1/FAILURE\nSep 23 [IPV6] pve01 systemd[1]: homelab-backup-audit.service: Failed with result 'exit-code'.\nSep 23 [IPV6] pve01 systemd[1]: Failed to start homelab-backup-audit.service - Homelab unified backup and DR audit.\nSep 23 [IPV6] pve01 systemd[1]: homelab-backup-audit.service: Consumed 2.367s CPU time, 148.3M memory peak.\nSep 23 [IPV6] pve01 systemd[1]: Starting homelab-backup-audit.service - Homelab unified backup and DR audit...\nSep 23 [IPV6] pve01 homelab-backup-audit[2372997]: TARGET|logical_id=app-filebrowser|kind=APP|state=FRESH|decision=SKIP_FRESH|legacy_age_seconds=16586|generation=app-filebrowser-pve01-20260923T143005Z\nSep 23 [IPV6] pve01 homelab-backup-audit[2372997]: TARGET|logical_id=app-hbarhub|kind=APP|state=FRESH|decision=SKIP_FRESH|legacy_age_seconds=16414|generation=app-hbarhub-pve01-20260923T143158Z\nSep 23 [IPV6] pve01 homelab-backup-audit[2372997]: TARGET|logical_id=app-immich|kind=APP|state=FRESH|decision=SKIP_FRESH|legacy_age_seconds=16514|generation=app-immich-pve01-20260923T143041Z\nSep 23 [IPV6] pve01 homelab-backup-audit[2372997]: TARGET|logical_id=app-skladchik-reports-monitor|kind=APP|state=FRESH|decision=SKIP_FRESH|legacy_age_seconds=16564|generation=app-skladchik-reports-monitor-pve01-20260923T143025Z\nSep 23 [IPV6] pve01 homelab-backup-audit[2372997]: TARGET|logical_id=control-bootstrap-v2|kind=CONTROL|state=FRESH|age_seconds=38415|generation=control-bootstrap-v2-pve01-20260916T120057Z\nSep 23 [IPV6] pve01 homelab-backup-audit[2372997]: TARGET|logical_id=vm110|kind=VM|state=FRESH|source=V2|v2_age_seconds=22042|legacy_age_seconds=1690029|replica_count=2|state_id=ct110|generation=v2-ct110-20260923T125546Z-34408ffe4182\nSep 23 [IPV6] pve01 homelab-backup-audit[2372997]: TARGET|logical_id=vm111|kind=VM|state=FRESH|source=V2|v2_age_seconds=19900|replica_count=2|state_id=ct111|generation=v2-ct111-20260923T133222Z-0d3f5fae9c35\nSep 23 [IPV6] pve01 homelab-backup-audit[2372997]: TARGET|logical_id=vm112|kind=VM|state=STALE|source=NONE|v2_age_seconds=72986|legacy_age_seconds=1728524|replica_count=2|state_id=ct112|generation=vm112-pve01-20260903T185707Z\nSep 23 [IPV6] pve01 homelab-backup-audit[2372997]: TARGET|logical_id=vm113|kind=VM|state=FRESH|source=V2|v2_age_seconds=71117|replica_count=2|state_id=ct113|generation=v2-ct113-20260922T231959Z-9fe7d084a330\nSep 23 [IPV6] pve01 homelab-backup-audit[2372997]: TARGET|logical_id=vm130|kind=VM|state=FRESH|source=V2|v2_age_seconds=63091|replica_count=2|state_id=vm130|generation=v2-vm130-20260923T011558Z-c9498d37d3d7\nSep 23 [IPV6] pve01 homelab-backup-audit[2372997]: TARGET|logical_id=vm150|kind=VM|state=FRESH|source=V2|v2_age_seconds=68773|legacy_age_seconds=1690145|replica_count=2|state_id=vm150|generation=v2-vm150-20260922T235842Z-71b8fcec59c9\nSep 23 [IPV6] pve01 homelab-backup-audit[2372997]: TARGET|logical_id=vm160|kind=VM|state=FRESH|source=V2|v2_age_seconds=61991|replica_count=2|state_id=vm160|generation=v2-vm160-20260923T014608Z-f76d7671af5a\nSep 23 [IPV6] pve01 homelab-backup-audit[2372997]: TARGET|logical_id=vm170|kind=VM|state=FRESH|source=V2|v2_age_seconds=53640|legacy_age_seconds=634766|replica_count=2|state_id=vm170|generation=v2-vm170-20260923T035958Z-923adacbc94b\nSep 23 [IPV6] pve01 homelab-backup-audit[2372997]: TARGET|logical_id=vm171|kind=VM|state=FRESH|source=V2|v2_age_seconds=60194|legacy_age_seconds=664298|replica_count=2|state_id=vm171|generation=v2-vm171-20260923T021946Z-4d6e4f578075\nSep 23 [IPV6] pve01 homelab-backup-audit[2372997]: TARGET|logical_id=vm190|kind=VM|state=FRESH|source=V2|v2_age_seconds=56111|replica_count=2|state_id=vm190|generation=v2-vm190-20260923T032759Z-314d9ba2ec35\nSep 23 [IPV6] pve01 homelab-backup-audit[2372997]: TARGET|logical_id=vm9130|kind=VM|state=FRESH|source=V2|v2_age_seconds=41206|replica_count=2|state_id=vm9130|generation=v2-vm9130-20260923T071932Z-c0492db6535b\nSep 23 [IPV6] pve01 homelab-backup-audit[2372997]: ISSUE|severity=HIGH|code=RPO_BREACH|logical_id=vm112|root_cause=NO_FRESH_V2_OR_SEALED_QUORUM\nSep 23 [IPV6] pve01 homelab-backup-audit[2372997]: ISSUE|severity=HIGH|code=LOCK_BUSY|logical_id=/run/homelab-backup-heavy.lock|root_cause=ACTIVE_OR_ORPHAN_LOCK\nSep 23 [IPV6] pve01 homelab-backup-audit[2372997]: SUMMARY|AUDIT_STATUS=OK|BACKUP_STATUS=RED|ACTIVE_COUNT=16|ISSUE_COUNT=2|CLOUD_A=OK|CLOUD_B=OK|CLOUD_B_DRILL=OK|DR_PROVEN=YES|SOURCE_OFFHOST=OK\nSep 23 [IPV6] pve01 systemd[1]: homelab-backup-audit.service: Main process exited, code=exited, status=1/FAILURE\nSep 23 [IPV6] pve01 systemd[1]: homelab-backup-audit.service: Failed with result 'exit-code'.\nSep 23 [IPV6] pve01 systemd[1]: Failed to start homelab-backup-audit.service - Homelab unified backup and DR audit.\nSep 23 [IPV6] pve01 systemd[1]: homelab-backup-audit.service: Consumed 2.438s CPU time, 148.4M memory peak.\nSep 23 [IPV6] pve01 systemd[1]: Starting homelab-backup-audit.service - Homelab unified backup and DR audit...\nSep 23 [IPV6] pve01 homelab-backup-audit[2466119]: TARGET|logical_id=app-filebrowser|kind=APP|state=FRESH|decision=SKIP_FRESH|legacy_age_seconds=18320|generation=app-filebrowser-pve01-20260923T143005Z\nSep 23 [IPV6] pve01 homelab-backup-audit[2466119]: TARGET|logical_id=app-hbarhub|kind=APP|state=FRESH|decision=SKIP_FRESH|legacy_age_seconds=18148|generation=app-hbarhub-pve01-20260923T143158Z\nSep 23 [IPV6] pve01 homelab-backup-audit[2466119]: TARGET|logical_id=app-immich|kind=APP|state=FRESH|decision=SKIP_FRESH|legacy_age_seconds=18248|generation=app-immich-pve01-20260923T143041Z\nSep 23 [IPV6] pve01 homelab-backup-audit[2466119]: TARGET|logical_id=app-skladchik-reports-monitor|kind=APP|state=FRESH|decision=SKIP_FRESH|legacy_age_seconds=18298|generation=app-skladchik-reports-monitor-pve01-20260923T143025Z\nSep 23 [IPV6] pve01 homelab-backup-audit[2466119]: TARGET|logical_id=control-bootstrap-v2|kind=CONTROL|state=FRESH|age_seconds=40150|generation=control-bootstrap-v2-pve01-20260916T120057Z\nSep 23 [IPV6] pve01 homelab-backup-audit[2466119]: TARGET|logical_id=vm110|kind=VM|state=FRESH|source=V2|v2_age_seconds=23777|legacy_age_seconds=1691763|replica_count=2|state_id=ct110|generation=v2-ct110-20260923T125546Z-34408ffe4182\nSep 23 [IPV6] pve01 homelab-backup-audit[2466119]: TARGET|logical_id=vm111|kind=VM|state=FRESH|source=V2|v2_age_seconds=21635|replica_count=2|state_id=ct111|generation=v2-ct111-20260923T133222Z-0d3f5fae9c35\nSep 23 [IPV6] pve01 homelab-backup-audit[2466119]: TARGET|logical_id=vm112|kind=VM|state=STALE|source=NONE|v2_age_seconds=74721|legacy_age_seconds=1730258|replica_count=2|state_id=ct112|generation=vm112-pve01-20260903T185707Z\nSep 23 [IPV6] pve01 homelab-backup-audit[2466119]: TARGET|logical_id=vm113|kind=VM|state=STALE|source=NONE|v2_age_seconds=72852|replica_count=2|state_id=ct113\nSep 23 [IPV6] pve01 homelab-backup-audit[2466119]: TARGET|logical_id=vm130|kind=VM|state=FRESH|source=V2|v2_age_seconds=64826|replica_count=2|state_id=vm130|generation=v2-vm130-20260923T011558Z-c9498d37d3d7\nSep 23 [IPV6] pve01 homelab-backup-audit[2466119]: TARGET|logical_id=vm150|kind=VM|state=FRESH|source=V2|v2_age_seconds=70508|legacy_age_seconds=1691879|replica_count=2|state_id=vm150|generation=v2-vm150-20260922T235842Z-71b8fcec59c9\nSep 23 [IPV6] pve01 homelab-backup-audit[2466119]: TARGET|logical_id=vm160|kind=VM|state=FRESH|source=V2|v2_age_seconds=63726|replica_count=2|state_id=vm160|generation=v2-vm160-20260923T014608Z-f76d7671af5a\nSep 23 [IPV6] pve01 homelab-backup-audit[2466119]: TARGET|logical_id=vm170|kind=VM|state=FRESH|source=V2|v2_age_seconds=55375|legacy_age_seconds=636500|replica_count=2|state_id=vm170|generation=v2-vm170-20260923T035958Z-923adacbc94b\nSep 23 [IPV6] pve01 homelab-backup-audit[2466119]: TARGET|logical_id=vm171|kind=VM|state=FRESH|source=V2|v2_age_seconds=61929|legacy_age_seconds=666032|replica_count=2|state_id=vm171|generation=v2-vm171-20260923T021946Z-4d6e4f578075\nSep 23 [IPV6] pve01 homelab-backup-audit[2466119]: TARGET|logical_id=vm190|kind=VM|state=FRESH|source=V2|v2_age_seconds=57846|replica_count=2|state_id=vm190|generation=v2-vm190-20260923T032759Z-314d9ba2ec35\nSep 23 [IPV6] pve01 homelab-backup-audit[2466119]: TARGET|logical_id=vm9130|kind=VM|state=FRESH|source=V2|v2_age_seconds=42941|replica_count=2|state_id=vm9130|generation=v2-vm9130-20260923T071932Z-c0492db6535b\nSep 23 [IPV6] pve01 homelab-backup-audit[2466119]: ISSUE|severity=HIGH|code=RPO_BREACH|logical_id=vm112|root_cause=NO_FRESH_V2_OR_SEALED_QUORUM\nSep 23 [IPV6] pve01 homelab-backup-audit[2466119]: ISSUE|severity=HIGH|code=RPO_BREACH|logical_id=vm113|root_cause=NO_FRESH_V2_OR_SEALED_QUORUM\nSep 23 [IPV6] pve01 homelab-backup-audit[2466119]: SUMMARY|AUDIT_STATUS=OK|BACKUP_STATUS=RED|ACTIVE_COUNT=16|ISSUE_COUNT=2|CLOUD_A=OK|CLOUD_B=OK|CLOUD_B_DRILL=OK|DR_PROVEN=YES|SOURCE_OFFHOST=OK\nSep 23 [IPV6] pve01 systemd[1]: homelab-backup-audit.service: Main process exited, code=exited, status=1/FAILURE\nSep 23 [IPV6] pve01 systemd[1]: homelab-backup-audit.service: Failed with result 'exit-code'.\nSep 23 [IPV6] pve01 systemd[1]: Failed to start homelab-backup-audit.service - Homelab unified backup and DR audit.\nSep 23 [IPV6] pve01 systemd[1]: homelab-backup-audit.service: Consumed 2.421s CPU time, 148M memory peak.\nSep 23 [IPV6] pve01 systemd[1]: Starting homelab-backup-audit.service - Homelab unified backup and DR audit...\nSep 23 [IPV6] pve01 homelab-backup-audit[2565382]: TARGET|logical_id=app-filebrowser|kind=APP|state=FRESH|decision=SKIP_FRESH|legacy_age_seconds=20177|generation=app-filebrowser-pve01-20260923T143005Z\nSep 23 [IPV6] pve01 homelab-backup-audit[2565382]: TARGET|logical_id=app-hbarhub|kind=APP|state=FRESH|decision=SKIP_FRESH|legacy_age_seconds=20005|generation=app-hbarhub-pve01-20260923T143158Z\nSep 23 [IPV6] pve01 homelab-backup-audit[2565382]: TARGET|logical_id=app-immich|kind=APP|state=FRESH|decision=SKIP_FRESH|legacy_age_seconds=20105|generation=app-immich-pve01-20260923T143041Z\nSep 23 [IPV6] pve01 homelab-backup-audit[2565382]: TARGET|logical_id=app-skladchik-reports-monitor|kind=APP|state=FRESH|decision=SKIP_FRESH|legacy_age_seconds=20155|generation=app-skladchik-reports-monitor-pve01-20260923T143025Z\nSep 23 [IPV6] pve01 homelab-backup-audit[2565382]: TARGET|logical_id=control-bootstrap-v2|kind=CONTROL|state=FRESH|age_seconds=42006|generation=control-bootstrap-v2-pve01-20260916T120057Z\nSep 23 [IPV6] pve01 homelab-backup-audit[2565382]: TARGET|logical_id=vm110|kind=VM|state=FRESH|source=V2|v2_age_seconds=25633|legacy_age_seconds=1693620|replica_count=2|state_id=ct110|generation=v2-ct110-20260923T125546Z-34408ffe4182\nSep 23 [IPV6] pve01 homelab-backup-audit[2565382]: TARGET|logical_id=vm111|kind=VM|state=FRESH|source=V2|v2_age_seconds=23491|replica_count=2|state_id=ct111|generation=v2-ct111-20260923T133222Z-0d3f5fae9c35\nSep 23 [IPV6] pve01 homelab-backup-audit[2565382]: TARGET|logical_id=vm112|kind=VM|state=STALE|source=NONE|v2_age_seconds=76577|legacy_age_seconds=1732115|replica_count=2|state_id=ct112|generation=vm112-pve01-20260903T185707Z\nSep 23 [IPV6] pve01 homelab-backup-audit[2565382]: TARGET|logical_id=vm113|kind=VM|state=STALE|source=NONE|v2_age_seconds=74708|replica_count=2|state_id=ct113\nSep 23 [IPV6] pve01 homelab-backup-audit[2565382]: TARGET|logical_id=vm130|kind=VM|state=FRESH|source=V2|v2_age_seconds=66682|replica_count=2|state_id=vm130|generation=v2-vm130-20260923T011558Z-c9498d37d3d7\nSep 23 [IPV6] pve01 homelab-backup-audit[2565382]: TARGET|logical_id=vm150|kind=VM|state=STALE|source=NONE|v2_age_seconds=72364|legacy_age_seconds=1693736|replica_count=2|state_id=vm150|generation=vm150-pve01-20260904T053517Z\nSep 23 [IPV6] pve01 homelab-backup-audit[2565382]: TARGET|logical_id=vm160|kind=VM|state=FRESH|source=V2|v2_age_seconds=65582|replica_count=2|state_id=vm160|generation=v2-vm160-20260923T014608Z-f76d7671af5a\nSep 23 [IPV6] pve01 homelab-backup-audit[2565382]: TARGET|logical_id=vm170|kind=VM|state=FRESH|source=V2|v2_age_seconds=57231|legacy_age_seconds=638357|replica_count=2|state_id=vm170|generation=v2-vm170-20260923T035958Z-923adacbc94b\nSep 23 [IPV6] pve01 homelab-backup-audit[2565382]: TARGET|logical_id=vm171|kind=VM|state=FRESH|source=V2|v2_age_seconds=63785|legacy_age_seconds=667889|replica_count=2|state_id=vm171|generation=v2-vm171-20260923T021946Z-4d6e4f578075\nSep 23 [IPV6] pve01 homelab-backup-audit[2565382]: TARGET|logical_id=vm190|kind=VM|state=FRESH|source=V2|v2_age_seconds=59702|replica_count=2|state_id=vm190|generation=v2-vm190-20260923T032759Z-314d9ba2ec35\nSep 23 [IPV6] pve01 homelab-backup-audit[2565382]: TARGET|logical_id=vm9130|kind=VM|state=FRESH|source=V2|v2_age_seconds=44797|replica_count=2|state_id=vm9130|generation=v2-vm9130-20260923T071932Z-c0492db6535b\nSep 23 [IPV6] pve01 homelab-backup-audit[2565382]: ISSUE|severity=HIGH|code=RPO_BREACH|logical_id=vm112|root_cause=NO_FRESH_V2_OR_SEALED_QUORUM\nSep 23 [IPV6] pve01 homelab-backup-audit[2565382]: ISSUE|severity=HIGH|code=RPO_BREACH|logical_id=vm113|root_cause=NO_FRESH_V2_OR_SEALED_QUORUM\nSep 23 [IPV6] pve01 homelab-backup-audit[2565382]: ISSUE|severity=HIGH|code=RPO_BREACH|logical_id=vm150|root_cause=NO_FRESH_V2_OR_SEALED_QUORUM\nBACKUP_AUDIT_FAILURE_EVIDENCE_END=1\nHEALTH_FILES_BEGIN=1\nHEALTH_FILE=cluster-admin-vm-mail-cloud-backup.txt|mtime=2026-07-09 08:07:47.213452046 +0300|sha256=e03b236f41f8d66f070ecd3234b3ff0836a44d972fd6f6abc721b9683fa22053\nSTATUS=OK TS=2026-07-09T05:07:47Z TYPE=cluster-admin-vm-mail-cloud-backup BAD=0 VMID=180 NODE=pve02 MODE=MAIL_CLOUD_2COPY_VERIFY_OK LOCAL_STATUS=OK REPORT=/var/lib/homelab-cluster-admin-vm-mail-cloud-backup/latest.md\nHEALTH_FILE=alertmanager-backup.txt|mtime=2026-07-09 10:50:17.826712693 +0300|sha256=f5a8a0ad29606ebbdcdcb52a997e9ee07ddec9ab7eab6bb0932c3f4e14eba97a\nSTATUS=OK TS=2026-07-09T07:50:16Z TYPE=alertmanager-backup JOB=homelab-pve01-app-alertmanager-20260709T075016Z SOURCE=alertmanager_config SIZE=1842 SHA256=58fb1b3467a962acff44bf75cb274e3a17c06afdbd3a25314a904f75c8048728 LOCAL_BACKUP_DELETED=true NOTE=service-readiness-backup-proof\nHEALTH_FILE=mealie-backup.txt|mtime=2026-07-09 19:17:35.017044807 +0300|sha256=8043626489c0b50dc30f11771e15940c0cb27c99cebdf482cf7dfba83571d074\nSTATUS=OK TS=2026-07-09T16:17:31Z TYPE=mealie-backup JOB=homelab-pve01-app-mealie-20260709T051713Z SOURCE=latest-appbackup SIZE=912487 LOCAL_BACKUP_DELETED=true JSON=/var/lib/homelab-backup/apps/latest-mealie.json NOTE=service-readiness-backup-proof\nHEALTH_FILE=karakeep-backup.txt|mtime=2026-07-09 22:28:35.572436986 +0300|sha256=79a25462c21ee9006462c6dc6fb7c2b21a4e7c297f585b770ed1e9a77e718230\nSTATUS=OK TS=2026-07-09T19:28:34Z TYPE=karakeep-backup BAD=0 APP=karakeep JOB=homelab-pve01-app-karakeep-20260709T051713Z SIZE=23140 LOCAL_BACKUP_DELETED=true JSON=/var/lib/homelab-backup/apps/latest-karakeep.json\nHEALTH_FILE=backup-restore-dashboard.txt|mtime=2026-07-03 00:45:30.941379161 +0300|sha256=51b95b7465dc040ff080c8a8341b23c9c0fdd90cfe0675ccd3f705b55590d815\nSTATUS=OK\nTS=20260702T214530Z\nTYPE=backup-restore-dashboard-rebuilt\nITEMS=13\nNOT_OK=0\nNOT_OK_LIST=none\nITEM name=backup.txt exists=true ok=true\nITEM name=restore-drill-index.txt exists=true ok=true\nITEM name=paperless-backup.txt exists=true ok=true\nITEM name=paperless-offhost.txt exists=true ok=true\nITEM name=paperless-restore.txt exists=true ok=true\nITEM name=npmplus-kuma-config-backup.txt exists=true ok=true\nITEM name=npmplus-kuma-config-offhost.txt exists=true ok=true\nITEM name=npmplus-kuma-config-restore.txt exists=true ok=true\nITEM name=netbird-peers.txt exists=true ok=true\nITEM name=crowdsec-capi-netbird-final.txt exists=true ok=true\nITEM name=adguard-rewrite-sync.txt exists=true ok=true\nITEM name=router-running-config-mail-cloud.txt exists=true ok=true\nITEM name=mail-cloud-restore-drill.txt exists=true ok=true\nHEALTH_FILE=backup-matrix-cluster-final-seal.txt|mtime=2026-07-13 20:51:32.422338540 +0300|sha256=dcd06c2ff207c2922fc68d4df4b42ffa4b6680785e532783772e764feb828aa1\nSTATUS=OK TS=2026-07-13T17:51:31Z TYPE=backup-matrix-cluster-final-seal BAD=0 CURRENT_STAGE_COMPLETE=YES MATRIX_STATUS=OK JOBS_TOTAL=35 JOBS_OK=31 JOBS_BAD=0 JOBS_UNMAPPED=0 JOBS_DISABLED=4 READ_WRITE_OVERLAP=0 VM_BACKUP_HEALTH=OK VM_FULL_BACKUP=OK EDGE_VM_CANONICAL_BACKUP=homelab-edge-vm-offhost-freshness.timer OLD_EDGE_TIMER_REENABLED=NO CLUSTER_PASSPORT=OK FINAL_READINESS=OK FINAL_READINESS_PASS=53 FINAL_READINESS_FAIL=0 OVERALL=OK CLUSTER_ADMIN=OK PROBE_OK=4 PROBE_BAD=0 WEBPANEL=OK FAILED_UNITS=0 CLUSTER_NODES=3 QUORATE=YES GENERATED_SYNC_HEAD=597119391e7b265b85fd9a76644e4e33af47e469 DESIRED_STATE_DIRTY=0 MATRIX_SHA256=7613bcb25d32e61fac114da13af81e38354ec22a012a6deeaa168ee4cc454f87 MATRIX_STATE_SHA256=14a31112aac5d577ae01df567921b093b2fedcf7189c4cc8f009216ee0749380 CLUSTER_PASSPORT_SHA256=1b8b924ae0e5ce6872fe5671318094ce494806c2db7efe28299b5ff49adc0d0d FINAL_READINESS_SHA256=ac5159fa8c497afe021b5cb232f08cfec0e1a4d516d8aae966ba03f7034f1412 OVERALL_SHA256=6ae529bfdf5811f686d3b53978f453ce4f8757c74fc1be105de219b13a61ba46 PRESEAL_SHA256=7ea1f710cd311bd51dba05a313b4c540f57d97407f008881e502fc213b4b3c99 EVIDENCE_MANIFEST_SHA256=b4c68312026e1328b208a74090891f4809ab31ec69069e94a3d6902429bf3566 BACKUP_RUN=NO RESTORE_RUN=NO CLOUD_WRITE_RUN=NO RETENTION_RUN=NO NEXT_ACTION=AUDIT_AND_MIGRATE_SPECIAL_VM_EMERGENCY_AND_NON_APP_BACKUP_FAMILIES EVIDENCE_ROOT=/var/lib/homelab-change-sets/backup-matrix-cluster-final-seal/20260713T175131Z\nHEALTH_FILE=actual-backup.txt|mtime=2026-07-09 15:53:45.248005185 +0300|sha256=72d1b31e22e22ac91af66e3d8ae46b20b3b9297809ace524eece22031458fb05\nSTATUS=OK TS=2026-07-09T12:53:43Z TYPE=actual-budget-backup JOB=homelab-pve01-app-actual-budget-20260709T051713Z SOURCE=latest-appbackup SIZE=2011 LOCAL_BACKUP_DELETED=true JSON=/var/lib/homelab-backup/apps/latest-actual-budget.json NOTE=service-readiness-backup-proof\nHEALTH_FILE=authentik-backup.txt|mtime=2026-08-30 21:08:08.459821635 +0300|sha256=8888712fdc8c5fd18ef0b691eb5d379fbd71af8f5dbda4e917b5a1493d12c046\nSTATUS=FAIL\nTS=2026-08-30T18:08:08Z\nTYPE=backup-matrix-generated\nJOB_ID=homelab_appbackup_postgres_batch\nJOB_NAME=homelab-appbackup-postgres-batch\nMATRIX_MODE=authoritative\nTIMER=homelab-appbackup-postgres-batch.timer\nSERVICE=homelab-appbackup-postgres-batch.service\nTIMER_ENABLED=not-found\nTIMER_ACTIVE=inactive\nSOURCE_KIND=systemd\nEVIDENCE_LEVEL=execution_only\nSERVICE_RESULT=success\nSERVICE_EXEC_MAIN_STATUS=0\nSOURCE_HEALTH=systemd:homelab-appbackup-postgres-batch.service\nSOURCE_STATUS=OK\nSOURCE_MTIME_UTC=2026-07-24T02:18:04Z\nAGE_SECONDS=3253804\nMAX_AGE_SECONDS=172800\nFRESHNESS=STALE\nLEGACY_SOURCE_PATH=/var/lib/homelab-health/authentik-backup.txt\nCANONICAL_JOB_ID=homelab_appbackup_postgres_batch\nCANONICAL_STATE=/var/lib/homelab-backup-matrix/state.json\nHEALTH_FILE=cluster-admin-stage3-postapply-backup.txt|mtime=2026-07-13 06:44:42.760995774 +0300|sha256=c00feef82c76522380776c84bf5bdb742ca2105c0dc2eeb1b596db2880facef9\nSTATUS=OK TS=2026-07-13T03:43:59Z TYPE=cluster-admin-stage3-postapply-backup BAD=0 COMMAND_PHYSICAL_LINES=1 PRODUCTION_CHANGED=POSTAPPLY_BACKUP_AND_CONTROLLED_COLLECTOR_RUN PRODUCTION_DATABASE_CHANGED=COLLECTOR_NORMAL_OPERATION_ONLY PRODUCTION_SCHEMA_CHANGED=NO LIVE_APPLICATION_CODE_CHANGED=NO DESIRED_STATE_CHANGED=NO SERVICE_RESTARTED=NO CLOUD_WRITE_RUN=YES_CONTROLLED_POSTAPPLY_BACKUP LIVE_APP_SHA256=a330a2d33ee5a8a57778e3541179e3faebcc0531600a504df744435e80a93b34 DESIRED_STATE_HEAD=b974057b536ece861299d0b230c06cc0e6702c39 APPBACKUP=VERIFIED APPBACKUP_JOB=homelab-pve01-app-cluster-admin-incident-engine-20260713T034400Z APPBACKUP_SIZE_BYTES=1791603 APPBACKUP_RESTORE_CHECK=OK APPBACKUP_ROUTING=VERIFIED COLLECTOR_CONTROL_RUN=OK COLLECTOR_RUN_ID=3113 CANONICAL_CONSISTENCY=0|0|0|0|0 INCIDENT_ENGINE=ACTIVE COLLECTOR_TIMER=ACTIVE VM180_FAILED_UNITS=0 READY_FOR_STAGE3_FINAL_SEAL=YES STAGE3_CLOSED=NO POSTAPPLY_ROOT=/var/lib/homelab-stage3-postapply/20260713T034359Z\nHEALTH_FILE=netbird-vps-trust.txt|mtime=2026-07-18 01:54:49.485000918 +0300|sha256=643c858c4b1cfa77a7914349ccb05434a94e27a358320b687658684665d5c1ac\nSTATUS=OK\nTS=2026-07-17T04:52:15Z\nTYPE=backup-matrix-generated\nJOB_ID=netbird_vps_backup\nJOB_NAME=netbird-vps-backup\nMATRIX_MODE=authoritative\nTIMER=netbird-vps-backup.timer\nSERVICE=netbird-vps-backup.service\nTIMER_ENABLED=enabled\nTIMER_ACTIVE=active\nSOURCE_KIND=health_file\nEVIDENCE_LEVEL=artifact_or_health\nSERVICE_RESULT=success\nSERVICE_EXEC_MAIN_STATUS=0\nSOURCE_HEALTH=/var/lib/homelab-backup-matrix/evidence/netbird_vps_backup/0-netbird-vps-trust.txt\nSOURCE_STATUS=OK\nSOURCE_MTIME_UTC=2026-07-07T15:24:59Z\nAGE_SECONDS=826036\nMAX_AGE_SECONDS=864000\nFRESHNESS=FRESH\nLEGACY_SOURCE_PATH=/var/lib/homelab-health/netbird-vps-trust.txt\nCANONICAL_JOB_ID=netbird_vps_backup\nCANONICAL_STATE=/var/lib/homelab-backup-matrix/state.json\nHEALTH_FILE=extended-appbackup.txt|mtime=2026-08-30 21:08:08.661414778 +0300|sha256=00167921437640beadbd7c5cc7323845b3893f85c161781d0899da9fba63f991\nSTATUS=FAIL\nTS=2026-08-30T18:08:08Z\nTYPE=backup-matrix-generated\nJOB_ID=homelab_extended_appbackup\nJOB_NAME=homelab-extended-appbackup\nMATRIX_MODE=authoritative\nTIMER=homelab-extended-appbackup.timer\nSERVICE=homelab-extended-appbackup.service\nTIMER_ENABLED=not-found\nTIMER_ACTIVE=inactive\nSOURCE_KIND=health_file\nEVIDENCE_LEVEL=artifact_or_health\nSERVICE_RESULT=success\nSERVICE_EXEC_MAIN_STATUS=0\nSOURCE_HEALTH=/var/lib/homelab-backup-matrix/evidence/homelab_extended_appbackup/0-extended-appbackup.txt\nSOURCE_STATUS=OK\nSOURCE_MTIME_UTC=2026-08-06T21:34:02Z\nAGE_SECONDS=2061246\nMAX_AGE_SECONDS=172800\nFRESHNESS=STALE\nLEGACY_SOURCE_PATH=/var/lib/homelab-health/extended-appbackup.txt\nCANONICAL_JOB_ID=homelab_extended_appbackup\nCANONICAL_STATE=/var/lib/homelab-backup-matrix/state.json\nHEALTH_FILE=healthchecks-backup.txt|mtime=2026-07-09 11:51:31.108386138 +0300|sha256=f2012e88374f88b98e419adb29ee39db9dd58540ed8f20d7894813eabad36301\nSTATUS=OK TS=2026-07-09T08:51:29Z TYPE=healthchecks-backup JOB=homelab-pve01-app-healthchecks-20260709T051713Z SOURCE=latest-appbackup SIZE=687 LOCAL_BACKUP_DELETED=true JSON=/var/lib/homelab-backup/apps/latest-healthchecks.json NOTE=service-readiness-backup-proof\nHEALTH_FILE=backup-scheduler-rpo-hardening.json|mtime=2026-08-10 13:51:49.375986623 +0300|sha256=e73927d669e1d066d03cd711449f9533ce018da041a86432c8550624a5a6c55f\n{\n \"status\": \"OK\",\n \"root_cause_count\": 0,\n \"production_changed\": true,\n \"cloud_changed\": false,\n \"backup_started\": false,\n \"applied_at_utc\": \"2026-08-10T10:51:49Z\",\n \"target_bytes\": true,\n \"persistent_stamps_rebased\": true,\n \"timers_started\": true,\n \"no_immediate_service_start\": true,\n \"next\": {\n \"vm\": \"Mon 2026-08-10 [IPV6] MSK\",\n \"app\": \"Mon 2026-08-10 [IPV6] MSK\"\n }\n}\nHEALTH_FILE=minio-backup.txt|mtime=2026-07-09 15:13:16.876111706 +0300|sha256=e4965b27706d848070d5ca6c2e974a14c520465f3b1575fda1ab671d6cf0e124\nSTATUS=OK TS=2026-07-09T12:13:14Z TYPE=minio-backup JOB=homelab-pve01-app-minio-20260709T051713Z SOURCE=latest-appbackup SIZE=4585 LOCAL_BACKUP_DELETED=true JSON=/var/lib/homelab-backup/apps/latest-minio.json NOTE=service-readiness-backup-proof\nHEALTH_FILE=backup-framework.txt|mtime=2026-08-30 21:08:08.850863900 +0300|sha256=192caeb6ecfd0d386aa32fcf838229854ea73f3abb7d46343cd8f1b4acf7b2ab\nSTATUS=FAIL\nTS=2026-08-30T18:08:08Z\nTYPE=backup-matrix-generated\nJOB_ID=homelab_verified_backup_health\nJOB_NAME=homelab-verified-backup-health\nMATRIX_MODE=authoritative\nTIMER=homelab-verified-backup-health.timer\nSERVICE=homelab-verified-backup-health.service\nTIMER_ENABLED=not-found\nTIMER_ACTIVE=inactive\nSOURCE_KIND=health_file\nEVIDENCE_LEVEL=artifact_or_health\nSERVICE_RESULT=success\nSERVICE_EXEC_MAIN_STATUS=0\nSOURCE_HEALTH=/var/lib/homelab-backup-matrix/evidence/homelab_verified_backup_health/1-latest-emergency-pack.json\nSOURCE_STATUS=OK\nSOURCE_MTIME_UTC=2026-08-06T20:59:56Z\nAGE_SECONDS=2063292\nMAX_AGE_SECONDS=172800\nFRESHNESS=STALE\nLEGACY_SOURCE_PATH=/var/lib/homelab-health/backup-framework.txt\nCANONICAL_JOB_ID=homelab_verified_backup_health\nCANONICAL_STATE=/var/lib/homelab-backup-matrix/state.json\nHEALTH_FILE=audiobookshelf-backup.txt|mtime=2026-07-09 21:22:25.670486930 +0300|sha256=fb81212dd08251ede49f921b0fb61b86a8f4446af80bdd42172acbbaf60b8e32\nSTATUS=OK TS=2026-07-09T18:22:24Z TYPE=audiobookshelf-backup BAD=0 APP=audiobookshelf JOB=homelab-pve01-app-audiobookshelf-20260709T051713Z SIZE=23094 LOCAL_BACKUP_DELETED=true JSON=/var/lib/homelab-backup/apps/latest-audiobookshelf.json\nHEALTH_FILE=filebrowser-backup.txt|mtime=2026-08-30 21:08:08.363475316 +0300|sha256=e8d06faa33f0c464b6074e5c121ec8aea9406d532097400a339764bcc71b009c\nSTATUS=FAIL\nTS=2026-08-30T18:08:08Z\nTYPE=backup-matrix-generated\nJOB_ID=filebrowser_backup\nJOB_NAME=filebrowser-backup\nMATRIX_MODE=authoritative\nTIMER=filebrowser-backup.timer\nSERVICE=filebrowser-backup.service\nTIMER_ENABLED=not-found\nTIMER_ACTIVE=inactive\nSOURCE_KIND=health_file\nEVIDENCE_LEVEL=artifact_or_health\nSERVICE_RESULT=success\nSERVICE_EXEC_MAIN_STATUS=0\nSOURCE_HEALTH=/var/lib/homelab-backup/apps/latest-filebrowser.json\nSOURCE_STATUS=OK\nSOURCE_MTIME_UTC=2026-08-06T21:28:05Z\nAGE_SECONDS=2061603\nMAX_AGE_SECONDS=259200\nFRESHNESS=STALE\nLEGACY_SOURCE_PATH=/var/lib/homelab-health/filebrowser-backup.txt\nCANONICAL_JOB_ID=filebrowser_backup\nCANONICAL_STATE=/var/lib/homelab-backup-matrix/state.json\nHEALTH_FILE=netbox-backup.txt|mtime=2026-07-09 13:23:55.693530989 +0300|sha256=0cf3e80b9d64332d56d5d6bb6fd0b09e3de366a6817b7cf4ce2fcca80f5ccd0c\nSTATUS=OK TS=2026-07-09T10:23:50Z TYPE=netbox-backup JOB=homelab-pve01-app-netbox-20260709T072101Z SOURCE=latest-appbackup SIZE=25860637 LOCAL_BACKUP_DELETED=true JSON=/var/lib/homelab-backup/apps/latest-netbox.json NOTE=service-readiness-backup-proof\nHEALTH_FILE=linkding-backup.txt|mtime=2026-07-09 20:00:19.838194359 +0300|sha256=cb3766255eec88a3a0a70447fb507ced5065345636fce5c4d760012f7e7944ab\nSTATUS=OK TS=2026-07-09T17:00:18Z TYPE=linkding-backup BAD=0 APP=linkding JOB=homelab-pve01-app-linkding-20260709T051713Z SIZE=40380 LOCAL_BACKUP_DELETED=true JSON=/var/lib/homelab-backup/apps/latest-linkding.json\nHEALTH_FILE=n8n-backup.txt|mtime=2026-07-09 22:39:24.237551932 +0300|sha256=723af1f45fc9e9e68e95cccc9747fb25b6bfef068c71060de1aaf260c0534774\nSTATUS=OK TS=2026-07-09T19:39:23Z TYPE=n8n-backup BAD=0 APP=n8n JOB=homelab-pve01-app-n8n-20260709T051713Z SIZE=92667 LOCAL_BACKUP_DELETED=true JSON=/var/lib/homelab-backup/apps/latest-n8n.json\nHEALTH_FILE=crypto-portfolio-backup.txt|mtime=2026-07-13 11:12:13.302397603 +0300|sha256=cf13d8b15a64fbb2fa412e32ce4eafdc732b045fe82e789928e768ace92361eb\nSTATUS=OK TS=2026-07-13T08:12:13Z TYPE=crypto-portfolio-backup APP=crypto-portfolio JOB=homelab-pve01-app-crypto-portfolio-20260713T075801Z PATH_REL=hb/a/cp-260713-075801 ARCHIVE_SIZE_BYTES=137496 FILE_COUNT=19 ARCHIVE_SHA256=d6d3f6a397c5e73b1df0ceaeeef0f313db557b163bc6b36f32e8a084e54e975a DATABASE_INTEGRITY=ok TWO_CLOUDS=OK LOCAL_STAGING_DELETED=YES\nHEALTH_FILE=backup-audit-v2.json|mtime=2026-09-23 23:47:36.447938462 +0300|sha256=b193b9fc04ed47f247d4c981468387ceff69cf1415e4295ec5b4de9e08d1b6fe\n{\"cloud\":{\"A\":{\"free\":592249131673,\"free_fraction\":0.538647,\"readable\":true,\"top_dirs\":27,\"total\":1099511627776,\"used\":507262496103},\"B\":{\"free\":592249067814,\"free_fraction\":0.538647,\"readable\":true,\"top_dirs\":27,\"total\":1099511627776,\"used\":507262559962}},\"cloud_b_restore_drill\":true,\"dr_proven\":true,\"generated_at_epoch\":1790196452,\"issues\":[{\"code\":\"RPO_BREACH\",\"logical_id\":\"vm150\",\"root_cause\":\"NO_FRESH_V2_OR_SEALED_QUORUM\",\"severity\":\"HIGH\"},{\"code\":\"SOURCE_OFFHOST_MISSING\",\"logical_id\":\"-\",\"root_cause\":\"CR_BRANCH_SHA_NOT_EXACT\",\"severity\":\"MEDIUM\"}],\"schema\":\"homelab-backup-audit-v3\",\"source\":{\"local_head\":\"6401ea1e99988b9d2550de544c9e45bd7e829bdd\",\"pushurl_disabled\":true,\"remote_head\":null},\"status\":\"RED\",\"targets\":[{\"decision\":\"SKIP_FRESH\",\"generation\":\"app-filebrowser-pve01-20260923T143005Z\",\"kind\":\"APP\",\"legacy_age_seconds\":22640,\"logical_id\":\"app-filebrowser\",\"state\":\"FRESH\"},{\"decision\":\"SKIP_FRESH\",\"generation\":\"app-hbarhub-pve01-20260923T143158Z\",\"kind\":\"APP\",\"legacy_age_seconds\":22468,\"logical_id\":\"app-hbarhub\",\"state\":\"FRESH\"},{\"decision\":\"SKIP_FRESH\",\"generation\":\"app-immich-pve01-20260923T143041Z\",\"kind\":\"APP\",\"legacy_age_seconds\":22568,\"logical_id\":\"app-immich\",\"state\":\"FRESH\"},{\"decision\":\"SKIP_FRESH\",\"generation\":\"app-skladchik-reports-monitor-pve01-20260923T143025Z\",\"kind\":\"APP\",\"legacy_age_seconds\":22618,\"logical_id\":\"app-skladchik-reports-monitor\",\"state\":\"FRESH\"},{\"age_seconds\":44470,\"generation\":\"control-bootstrap-v2-pve01-20260916T120057Z\",\"kind\":\"CONTROL\",\"logical_id\":\"control-bootstrap-v2\",\"state\":\"FRESH\"},{\"generation\":\"v2-ct110-20260923T125546Z-34408ffe4182\",\"kind\":\"VM\",\"legacy_age_seconds\":1696083,\"logical_id\":\"vm110\",\"replica_count\":2,\"source\":\"V2\",\"state\":\"FRESH\",\"state_id\":\"ct110\",\"v2_age_seconds\":28097},{\"generation\":\"v2-ct111-20260923T133222Z-0d3f5fae9c35\",\"kind\":\"VM\",\"legacy_age_seconds\":null,\"logical_id\":\"vm111\",\"replica_count\":2,\"source\":\"V2\",\"state\":\"FRESH\",\"state_id\":\"ct111\",\"v2_age_seconds\":25955},{\"generation\":\"v2-ct112-20260923T200600Z-c1b47cc67dad\",\"kind\":\"VM\",\"legacy_age_seconds\":1734578,\"logical_id\":\"vm112\",\"replica_count\":2,\"source\":\"V2\",\"state\":\"FRESH\",\"state_id\":\"ct112\",\"v2_age_seconds\":2422},{\"generation\":\"v2-ct113-20260923T203800Z-bd913a8e8930\",\"kind\":\"VM\",\"legacy_age_seconds\":null,\"logical_id\":\"vm113\",\"replica_count\":2,\"source\":\"V2\",\"state\":\"FRESH\",\"state_id\":\"ct113\",\"v2_age_seconds\":497},{\"generation\":\"v2-vm130-20260923T011558Z-c9498d37d3d7\",\"kind\":\"VM\",\"legacy_age_seconds\":null,\"logical_id\":\"vm130\",\"replica_count\":2,\"source\":\"V2\",\"state\":\"FRESH\",\"state_id\":\"vm130\",\"v2_age_seconds\":69146},{\"generation\":\"vm150-pve01-20260904T053517Z\",\"kind\":\"VM\",\"legacy_age_seconds\":1696199,\"logical_id\":\"vm150\",\"replica_count\":2,\"source\":\"NONE\",\"state\":\"STALE\",\"state_id\":\"vm150\",\"v2_age_seconds\":74828},{\"generation\":\"v2-vm160-20260923T014608Z-f76d7671af5a\",\"kind\":\"VM\",\"legacy_age_seconds\":null,\"logical_id\":\"vm160\",\"replica_count\":2,\"source\":\"V2\",\"state\":\"FRESH\",\"state_id\":\"vm160\",\"v2_age_seconds\":68046},{\"generation\":\"v2-vm170-20260923T035958Z-923adacbc94b\",\"kind\":\"VM\",\"legacy_age_seconds\":640820,\"logical_id\":\"vm170\",\"replica_count\":2,\"source\":\"V2\",\"state\":\"FRESH\",\"state_id\":\"vm170\",\"v2_age_seconds\":59695},{\"generation\":\"v2-vm171-20260923T021946Z-4d6e4f578075\",\"kind\":\"VM\",\"legacy_age_seconds\":670352,\"logical_id\":\"vm171\",\"replica_count\":2,\"source\":\"V2\",\"state\":\"FRESH\",\"state_id\":\"vm171\",\"v2_age_seconds\":66249},{\"generation\":\"v2-vm190-20260923T032759Z-314d9ba2ec35\",\"kind\":\"VM\",\"legacy_age_seconds\":null,\"logical_id\":\"vm190\",\"replica_count\":2,\"source\":\"V2\",\"state\":\"FRESH\",\"state_id\":\"vm190\",\"v2_age_seconds\":62166},{\"generation\":\"v2-vm9130-20260923T071932Z-c0492db6535b\",\"kind\":\"VM\",\"legacy_age_seconds\":null,\"logical_id\":\"vm9130\",\"replica_count\":2,\"source\":\"V2\",\"state\":\"FRESH\",\"state_id\":\"vm9130\",\"v2_age_seconds\":47261}]}\nHEALTH_FILE=netbird-old-usa-poweroff-observation.txt|mtime=2026-08-22 05:33:01.774623609 +0300|sha256=0307f121dba0190d02875321ed3ca2c433f8a2902678f46814d39cdc476a4a00\nSTATUS=OBSERVING\nSTART_EPOCH=1787365981\nSTART_UTC=2026-08-22T02:33:01Z\nMIN_SECONDS=86400\nMAX_SECONDS=172800\nNEW_IP=[IP]\nOLD_USA_PROVIDER_DELETION=USER_ATTESTED\nOLD_PUBLIC_IP_REACHABILITY=NOT_USED_AS_HOST_IDENTITY\nNEW_OPERATOR_KEY_ONLY=PROVEN\nNEW_NETBIRD_STACK=3_OF_3\nNEW_PUBLIC_PLANE=GREEN\nFINAL_RETIREMENT=PENDING_OBSERVATION\nHEALTH_FILE=crypto-portfolio-remove-grt-prechange-backup.txt|mtime=2026-07-13 08:10:44.727781612 +0300|sha256=59de8a4e38310ef8236d167be536aa2f0f1d0fa3dd0ad3ac7fd49e67ed6206f3\nSTATUS=OK TS=2026-07-13T05:10:13Z TYPE=crypto-portfolio-remove-grt-prechange-backup BAD=0 COMMAND_PHYSICAL_LINES=1 PRODUCTION_CHANGED=ROLLBACK_SNAPSHOT_AND_BACKUP_ONLY PRODUCTION_DATABASE_CHANGED=NO DESIRED_STATE_CHANGED=NO_BY_THIS_COMMAND SERVICE_RESTARTED=NO PRODUCTION_CONTAINER_RECREATED=NO CLOUD_WRITE_RUN=YES_CONTROLLED_PRECHANGE_BACKUP RETENTION_RUN=NO PRODUCTION_APPLY=NO TARGET_HOST=core-apps.gram1.ru DESIRED_STATE_HEAD=80763a6c62b94cde4f3293f3b529980f76919d25 CANDIDATE_ROOT=/var/lib/homelab-change-sets/crypto-portfolio-remove-grt-candidate-v3/20260713T042046Z INPUT_APP_SHA256=684ad884119554503db0ed36252aeb017c0c04b54729f30c51e66a7ab293f8bd INPUT_WORKER_SHA256=40a1c81111d7097120125b140683a8604abcb4051db5073ab8af708a11854038 INPUT_INDEX_SHA256=2b1ab459cabf3050af1db4a7b293fff7c2a9ac18bf4a14faa1efc70a3a3f63e3 EXACT_LOCAL_ROLLBACK=OK ROLLBACK_PATH=/var/lib/homelab-incidents/crypto-remove-grt-prechange-20260713T051013Z ROLLBACK_MANIFEST_SHA256=6cc505f7e13a02b0ddfc9fd3a9700903d7aaf70d8c8ae3f630a4fea69aeb4b1f ROLLBACK_DATABASE_INTEGRITY=ok ROLLBACK_GRT_ROWS=1|1|729 TWO_CLOUD_BACKUP=OK BACKUP_JOB=homelab-pve01-app-crypto-portfolio-20260713T051015Z BACKUP_ARCHIVE_SIZE_BYTES=172776 BACKUP_FILE_COUNT=19 BACKUP_DATABASE_INTEGRITY=ok BACKUP_LOCAL_STAGING_DELETED=YES RESTORE_CHECK=OK_BOTH_CLOUDS PRODUCTION_SOURCE_HASHES_UNCHANGED=YES PRODUCTION_CONTAINER_IDS_AND_RESTARTS_UNCHANGED=YES PRODUCTION_DATABASE_INTEGRITY=ok PRODUCTION_GRT_STATE_PRESERVED=YES PUBLIC_PRODUCTION_STATE_PRESERVED=YES READY_FOR_GRT_PRODUCTION_APPLY=YES PRECHANGE_ROOT=/var/lib/homelab-change-sets/crypto-portfolio-remove-grt-prechange-backup/20260713T051013Z\nHEALTH_FILE=beszel-backup.txt|mtime=2026-07-09 11:38:36.118613080 +0300|sha256=951a25a876714a6b36934a96ae7f85bab06b480633c052b0850ba02d5e0e106c\nSTATUS=OK TS=2026-07-09T08:38:10Z TYPE=beszel-backup JOB=homelab-pve01-app-beszel-20260709T083810Z SOURCE=beszel_stack_data SIZE=20030 SHA256=a3d7069dce01b7e7b336268cfe383dc1b0842a974bd5dc9f455442ed86c912e4 LOCAL_BACKUP_DELETED=true NOTE=service-readiness-backup-proof\nHEALTH_FILE=gotify-backup.txt|mtime=2026-07-09 23:39:37.911441599 +0300|sha256=e86e933ad71df820106148796cdccbde701c986034c7e49684f3d9e84bc0d17f\nSTATUS=OK TS=2026-07-09T20:39:29Z TYPE=gotify-backup BAD=0 APP=gotify JOB=homelab-pve01-app-gotify-20260709T072202Z SIZE=44431 LOCAL_BACKUP_DELETED=true JSON=/var/lib/homelab-backup/apps/latest-gotify.json\nHEALTH_FILE=minio-console-backup.txt|mtime=2026-07-09 15:13:16.876111706 +0300|sha256=e4965b27706d848070d5ca6c2e974a14c520465f3b1575fda1ab671d6cf0e124\nSTATUS=OK TS=2026-07-09T12:13:14Z TYPE=minio-backup JOB=homelab-pve01-app-minio-20260709T051713Z SOURCE=latest-appbackup SIZE=4585 LOCAL_BACKUP_DELETED=true JSON=/var/lib/homelab-backup/apps/latest-minio.json NOTE=service-readiness-backup-proof\nHEALTH_FILE=netbird-vps-restore.txt|mtime=2026-06-17 10:54:28.050678269 +0300|sha256=800a5462a6173c8749059462e39a3263cae2b4db012c1dc3ad336aa3429ab2cb\nSTATUS=OK\nSERVICE=netbird-vps-restore-validation\nTIMESTAMP_UTC=20260617T075428Z\nLATEST=20260617T074922Z\nARCHIVE=/mnt/staging/netbird-vps-backups/snapshots/20260617T074922Z/netbird-vps-backup.tgz\nSIZE=32M\nSHA256=8e0c6538268b4e5dc463e008e59acb9d77873873915bec74ac2a85a9d1523a84\nTOP_ARCHIVE_ENTRIES=9\nROOT_NETBIRD_ENTRIES=7\nCHECKS=compose,config,dashboard_env\nHEALTH_FILE=bookstack-backup.txt|mtime=2026-07-09 20:58:33.528404671 +0300|sha256=80a1b54e2027c39a6b2c298e558d1352b976e0cd1795cf8f26ae9249b8c2e1dd\nSTATUS=OK TS=2026-07-09T17:58:30Z TYPE=bookstack-backup BAD=0 APP=bookstack JOB=homelab-pve01-app-bookstack-20260709T024714Z SIZE=18108 LOCAL_BACKUP_DELETED=true JSON=/var/lib/homelab-backup/apps/latest-bookstack.json\nHEALTH_FILE=backup-last-run.log|mtime=2026-07-24 07:04:41.713444052 +0300|sha256=8618b97d2ffa65b7cc2218710c2add4283285038eaca633a7bffbda898d1e5c7\n===== HOMELAB BACKUP HEALTHCHECK =====\n2026-07-24T07:04:28+03:00\n\nLATEST=/mnt/staging/vzdump/auto-full-backup-20260719T004537Z\nAGE_HOURS=123\nMAX_AGE_HOURS=192\n\n===== REQUIRED FILES =====\nOK: pve01/vzdump-lxc-110-*.tar.zst\nOK: pve01/vzdump-lxc-112-*.tar.zst\nOK: pve01/vzdump-qemu-150-*.vma.zst\nOK: pve02/vzdump-lxc-111-*.tar.zst\nOK: pve02/vzdump-lxc-113-*.tar.zst\nOK: pve03/vzdump-qemu-130-*.vma.zst\nOK: MASTER_SHA256SUMS.txt\nOK: README_AUTO_FULL_BACKUP.txt\n\n===== SIZE CHECK =====\nSIZE_BYTES=19102676192\nSIZE_GB_APPROX=17\n\n===== SHA256 CHECK =====\npve01/SHA256SUMS.txt: OK\npve01/vzdump-lxc-110-2026_07_19-03_45_38.log: OK\npve01/vzdump-lxc-110-2026_07_19-03_45_38.tar.zst: OK\npve01/vzdump-lxc-112-2026_07_19-03_45_47.log: OK\npve01/vzdump-lxc-112-2026_07_19-03_45_47.tar.zst: OK\npve01/vzdump-qemu-150-2026_07_19-03_45_58.log: OK\npve01/vzdump-qemu-150-2026_07_19-03_45_58.vma.zst: OK\npve02/SHA256SUMS.txt: OK\npve02/vzdump-lxc-111-2026_07_19-03_47_26.log: OK\npve02/vzdump-lxc-111-2026_07_19-03_47_26.tar.zst: OK\npve02/vzdump-lxc-113-2026_07_19-03_47_43.log: OK\npve02/vzdump-lxc-113-2026_07_19-03_47_43.tar.zst: OK\npve03/SHA256SUMS.txt: OK\npve03/vzdump-qemu-130-2026_07_19-03_48_01.log: OK\npve03/vzdump-qemu-130-2026_07_19-03_48_01.vma.zst: OK\nREADME_AUTO_FULL_BACKUP.txt: OK\n\n===== STORAGE SPACE =====\nFilesystem Type Size Used Avail Use% Mounted on\n/dev/mapper/pve-homelab--staging ext4 295G 148G 132G 53% /mnt/staging\n/dev/mapper/pve-root ext4 98G 41G 53G 44% /\n\nSTATUS=OK\nREASON=latest backup exists, fresh, complete, and sha256 verified\nHEALTH_FILE=cluster-admin-stage2-preapply-backup.txt|mtime=2026-07-13 04:24:04.995755658 +0300|sha256=82a0a705c46b2cf757dc37ce6a06a981c4cbe3ea3d580459ac7f8c6ae17a153d\nSTATUS=OK TS=2026-07-13T01:23:18Z TYPE=cluster-admin-stage2-preapply-backup BAD=0 PREAPPLY_GENERATION=V2_DYNAMIC PRODUCTION_CHANGED=BACKUP_ARTIFACTS_ONLY PRODUCTION_DATABASE_CHANGED=NO PRODUCTION_SCHEMA_CHANGED=NO INCIDENT_ENGINE_RESTARTED=NO COLLECTOR_ONESHOT_RUN=NO CLOUD_WRITE_RUN=YES_CONTROLLED_BACKUP_ONLY ROLLBACK_POINT=VERIFIED DIRECT_DB_SNAPSHOT=VERIFIED DIRECT_DB_RESTORE_TEST=OK DIRECT_DB_DUMP_SIZE_BYTES=1818083 DIRECT_DB_DUMP_SHA256=c0fe2161bb6e61b423bc72d809bdd579ac29dc4f7bac79ba4c493bfa9f8b1d5e PRECHANGE_COLLECTOR_SHA256=a83abde2290d8392b1c631122671bcb680f9271d0b576fafacb8876d3c8034f1 CANDIDATE_COLLECTOR_SHA256=5dda66a1878046223af9c28a393129b44991a8083b430a76302767b460e7cbbe FOUNDATION_COUNTS=84|21|1|227|26|0 DYNAMIC_INVARIANTS=0|0|21|21|0|0|0|0|0|0 RETIRED_PLACEHOLDERS_ALLOWED=YES FIXED_TOTAL_ENTITIES_REQUIRED=NO FIXED_TOTAL_ALIASES_REQUIRED=NO APPBACKUP_JOB=homelab-pve01-app-cluster-admin-incident-engine-20260713T012320Z APPBACKUP_STATUS=success APPBACKUP_RESTORE_CHECK=OK APPBACKUP_ROUTING_DOCKGE_UNCHANGED=YES MIGRATION_PATCH_REQUIRED=NO COLLECTOR_PATCH_REQUIRED=NO APPLY_VERIFIER_PATCH_REQUIRED=YES PREFLIGHT_TRANSACTION_ROLLBACK_REQUIRED=YES ROLLBACK_AFTER_COMMITTED_SYNC=PRESERVE_APPLIED_STATE_AND_DIAGNOSE PRODUCTION_UNCHANGED=YES OVERALL=OK DUTY_ADMIN=OK READY_FOR_DYNAMIC_PRODUCTION_APPLY=YES DESIRED_STATE_HEAD=4d0d03395d2baf4024ad7c9d5870f21bdaee9f99 PREAPPLY_ROOT=/var/lib/homelab-stage2-preapply-v2/20260713T012318Z REMOTE_SNAPSHOT_DIR=/var/lib/cluster-admin-stage2-preapply-v2/20260713T012318Z CONTRACT=/var/lib/homelab-change-sets/stage2-dynamic-redryrun/20260713T012006Z/dynamic-acceptance-contract.json EVIDENCE_MANIFEST=/var/lib/homelab-stage2-preapply-v2/20260713T012318Z/SHA256SUMS\nHEALTH_FILE=blackbox-exporter-two-cloud-backup.txt|mtime=2026-07-10 22:20:26.139732318 +0300|sha256=ab822f689cc4b7c38a0eed3c6a3164993ceb54fa05033250c68d934d3df0f259\nSTATUS=OK TS=2026-07-10T19:20:26Z TYPE=blackbox-exporter-two-cloud-backup BAD=0 JOB=homelab-pve01-app-blackbox-exporter-20260710T191958Z ARCHIVE_SIZE_BYTES=592 LOCAL_STAGING_DELETED=true OFFHOST=OK RESTORE=OK\nHEALTH_FILE=backup.json|mtime=2026-07-24 07:04:41.723802122 +0300|sha256=550e7a326c46e3e765a6db01da2ddcaddd83808adfa638d980685286bebb45cb\n{\n \"status\": \"OK\",\n \"reason\": \"latest backup exists, fresh, complete, and sha256 verified\",\n \"latest\": \"/mnt/staging/vzdump/auto-full-backup-20260719T004537Z\",\n \"age_hours\": \"123\",\n \"start\": \"2026-07-24T07:04:28+03:00\",\n \"end\": \"2026-07-24T07:04:41+03:00\",\n \"rc\": 0\n}\nHEALTH_FILE=cluster-admin-backup-webpanel-final-seal.txt|mtime=2026-07-09 08:24:54.072015305 +0300|sha256=9739e2a0d2bbb859b25255fd7acae984ee05a16fb79cd9e59ff81529c4acf3fd\nSTATUS=OK TS=2026-07-09T05:24:54Z TYPE=cluster-admin-backup-webpanel-final-seal BAD=0 VMID=180 URL=http://[IP]:8080/ BACKUP=MAIL_CLOUD_2COPY_VERIFY_OK WEBPANEL=OK WEBPANEL_SYNC=OK OVERALL_HARDGATE=YES FINAL_READINESS_HARDGATE=YES FAILED_UNITS=pve01:0,pve02:0,pve03:0,edge:0 DIRTY_COUNT=0 REPORT=/var/lib/homelab-ideal-cluster/cluster-admin-backup-webpanel-final-seal-latest.md\nHEALTH_FILE=immich-backup.txt|mtime=2026-07-09 23:09:25.074550210 +0300|sha256=8ef28f515f12888a7b1ff5b949d09b0eef90c1fc11da523b388ddd905167b52d\nSTATUS=OK TS=2026-07-09T20:09:12Z TYPE=immich-backup BAD=0 APP=immich JOB=homelab-pve01-app-immich-20260709T072002Z SIZE=328392286 LOCAL_BACKUP_DELETED=true JSON=/var/lib/homelab-backup/apps/latest-immich.json MEDIA_BACKUP_HEALTH=/var/lib/homelab-health/immich-media-backup.txt\nHEALTH_FILE=grafana-backup.txt|mtime=2026-07-09 10:46:00.905608392 +0300|sha256=1347e0bba67dd249caa8d4e46faf4d6bb2fd573255c1f78449995f97eea30014\nSTATUS=OK TS=2026-07-09T07:46:00Z TYPE=grafana-backup JOB=homelab-pve01-app-grafana-20260709T074600Z SOURCE=grafana_sqlite_backup_plus_provisioning SIZE=129929 SHA256=582537912f306666f04d9e855491f7275bdb31b39b4c9b0156444a297f8c1641 LOCAL_BACKUP_DELETED=true NOTE=service-readiness-backup-proof\nHEALTH_FILE=appbackupctl-routing.txt|mtime=2026-07-13 03:29:40.002577636 +0300|sha256=d8ddbdb46b42bcefb4f03b50ef4b88bdc5e3cd1cea087dd035fd90fd78e1a252\nSTATUS=OK TS=2026-07-13T00:29:36Z TYPE=appbackupctl-routing BAD=0 RESULT=APPLIED PRODUCTION_CHANGED=BACKUP_CLI_ROUTING_AND_DESIRED_STATE_ONLY PRODUCTION_DATABASE_CHANGED=NO PRODUCTION_SCHEMA_CHANGED=NO SERVICE_RESTARTED=NO CLOUD_WRITE_RUN=NO DEFAULT_ROUTE=dockge-stacks CLUSTER_ADMIN_ROUTE=homelab-cluster-admin-incident-engine-backup.service UNSUPPORTED_TARGET_POLICY=REFUSE_RC_2 ROUTE_TEST_DOCKGE=OK ROUTE_TEST_CLUSTER_ADMIN=OK ROUTE_TEST_DEFAULT=OK ROUTE_TEST_UNSUPPORTED=OK FRAMEWORK_CHECK=OK UNSUPPORTED_ROUTE_BACKUP_CREATED=NO LIVE_MATCH_DESIRED_STATE=YES PVE01_FAILED_UNITS=0 OVERALL=OK DUTY_ADMIN=OK STAGE1_CLOSED=YES STAGE2_READY=YES OLD_HEAD=bd5eb856a8d61a5fb83c2bcaf74cb8ac98f075c5 NEW_HEAD=4d0d03395d2baf4024ad7c9d5870f21bdaee9f99 CHANGE_SET=/var/lib/homelab-change-sets/appbackupctl-routing/20260713T002936Z\nHEALTH_FILE=vaultwarden-backup.txt|mtime=2026-08-30 21:08:08.480614603 +0300|sha256=ba016970c3d244955b82977178290fea97126dbcc2a44e4758ecd28ad720a499\nSTATUS=FAIL\nTS=2026-08-30T18:08:08Z\nTYPE=backup-matrix-generated\nJOB_ID=homelab_appbackup_sqlite_batch\nJOB_NAME=homelab-appbackup-sqlite-batch\nMATRIX_MODE=authoritative\nTIMER=homelab-appbackup-sqlite-batch.timer\nSERVICE=homelab-appbackup-sqlite-batch.service\nTIMER_ENABLED=not-found\nTIMER_ACTIVE=inactive\nSOURCE_KIND=systemd\nEVIDENCE_LEVEL=execution_only\nSERVICE_RESULT=success\nSERVICE_EXEC_MAIN_STATUS=0\nSOURCE_HEALTH=systemd:homelab-appbackup-sqlite-batch.service\nSOURCE_STATUS=OK\nSOURCE_MTIME_UTC=2026-07-24T01:46:28Z\nAGE_SECONDS=3255700\nMAX_AGE_SECONDS=172800\nFRESHNESS=STALE\nLEGACY_SOURCE_PATH=/var/lib/homelab-health/vaultwarden-backup.txt\nCANONICAL_JOB_ID=homelab_appbackup_sqlite_batch\nCANONICAL_STATE=/var/lib/homelab-backup-matrix/state.json\nHEALTH_FILE=hapusya-site-preapply-backup.txt|mtime=2026-07-13 14:03:06.015223430 +0300|sha256=00e05707281a8d6e841ec0f6d2b8c95ee0f9fbc090ab90b718e85530d41854f8\nSTATUS=OK TS=2026-07-13T11:03:06Z TYPE=hapusya-site-preapply-backup BAD=0 NPMPLUS_JOB=homelab-pve01-app-npmplus-edge-20260713T110238Z TWO_CLOUDS=OK RESTORE_CHECK=OK ROLLBACK=/var/lib/homelab-incidents/hapusya-site-preapply-20260713T110305Z ROLLBACK_MANIFEST_SHA256=6b4020425c5d6c581c48a5dc95b20d54fd7906468e4ba8d27c0f0325e2df81a3 CANDIDATE_BROWSER=OK BROWSER_RUNS=4 PRODUCTION_CHANGED=NO\nHEALTH_FILE=pve01-backup.txt|mtime=2026-07-09 15:29:41.985138298 +0300|sha256=2b8ca75eeec121d7e2b0c6a309d43798da5f864b57fef327f37a56b9ced3b5a8\nSTATUS=OK TS=2026-07-09T12:29:38Z TYPE=proxmox-pve01-backup JOB=homelab-pve01-app-proxmox-pve01-20260709T122938Z SOURCE=safe_pve01_config_facts SIZE=4314 LOCAL_BACKUP_DELETED=true JSON=/var/lib/homelab-backup/apps/latest-proxmox-pve01.json NOTE=service-readiness-backup-proof\nHEALTH_FILE=backup-observability-history.jsonl|mtime=2026-09-21 07:07:33.804291373 +0300|sha256=24fa8197552d64778c83071fb94a64accd398075ce5d1e675dcdf6c97a84c36b\n{\"clouds\":[{\"free_bytes\":792267937635,\"remote\":\"pve01-mail-01-crypt\",\"total_bytes\":1099511627776,\"used_bytes\":307243690141},{\"free_bytes\":792267873776,\"remote\":\"pve01-mail-02-crypt\",\"total_bytes\":1099511627776,\"used_bytes\":307243754000}],\"observed_at_epoch\":1788438952,\"schema\":\"homelab-backup-observability-snapshot-v1\",\"targets\":[{\"archive_size_bytes\":474143241,\"id\":\"110\",\"kind\":\"VM\",\"logical_id\":\"vm110\"},{\"archive_size_bytes\":369191552,\"id\":\"111\",\"kind\":\"VM\",\"logical_id\":\"vm111\"},{\"archive_size_bytes\":223627414,\"id\":\"112\",\"kind\":\"VM\",\"logical_id\":\"vm112\"},{\"archive_size_bytes\":221521137,\"id\":\"113\",\"kind\":\"VM\",\"logical_id\":\"vm113\"},{\"archive_size_bytes\":10622796310,\"id\":\"130\",\"kind\":\"VM\",\"logical_id\":\"vm130\"},{\"archive_size_bytes\":890940072,\"id\":\"150\",\"kind\":\"VM\",\"logical_id\":\"vm150\"},{\"archive_size_bytes\":3469023634,\"id\":\"160\",\"kind\":\"VM\",\"logical_id\":\"vm160\"},{\"archive_size_bytes\":7824247513,\"id\":\"170\",\"kind\":\"VM\",\"logical_id\":\"vm170\"},{\"archive_size_bytes\":2026787893,\"id\":\"171\",\"kind\":\"VM\",\"logical_id\":\"vm171\"},{\"archive_size_bytes\":1176306536,\"id\":\"180\",\"kind\":\"VM\",\"logical_id\":\"vm180\"},{\"archive_size_bytes\":1551688736,\"id\":\"190\",\"kind\":\"VM\",\"logical_id\":\"vm190\"},{\"archive_size_bytes\":11005893463,\"id\":\"9130\",\"kind\":\"VM\",\"logical_id\":\"vm9130\"},{\"archive_size_bytes\":383573170,\"id\":\"200\",\"kind\":\"VM\",\"logical_id\":\"vm200\"},{\"archive_size_bytes\":3193,\"id\":\"filebrowser\",\"kind\":\"APP\",\"logical_id\":\"app-filebrowser\"},{\"archive_size_bytes\":152745,\"id\":\"skladchik-reports-monitor\",\"kind\":\"APP\",\"logical_id\":\"app-skladchik-reports-monitor\"},{\"archive_size_bytes\":270193536,\"id\":\"immich\",\"kind\":\"APP\",\"logical_id\":\"app-immich\"},{\"archive_size_bytes\":75298798,\"id\":\"hbarhub\",\"kind\":\"APP\",\"logical_id\":\"app-hbarhub\"},{\"archive_size_bytes\":8959200,\"id\":\"control-bootstrap-v2\",\"kind\":\"CONTROL\",\"logical_id\":\"control-bootstrap-v2\"}]}\n{\"clouds\":[{\"free_bytes\":792267937635,\"remote\":\"pve01-mail-01-crypt\",\"total_bytes\":1099511627776,\"used_bytes\":307243690141},{\"free_bytes\":792267873776,\"remote\":\"pve01-mail-02-crypt\",\"total_bytes\":1099511627776,\"used_bytes\":307243754000}],\"observed_at_epoch\":1788442632,\"schema\":\"homelab-backup-observability-snapshot-v1\",\"targets\":[{\"archive_size_bytes\":474143241,\"id\":\"110\",\"kind\":\"VM\",\"logical_id\":\"vm110\"},{\"archive_size_bytes\":369191552,\"id\":\"111\",\"kind\":\"VM\",\"logical_id\":\"vm111\"},{\"archive_size_bytes\":223627414,\"id\":\"112\",\"kind\":\"VM\",\"logical_id\":\"vm112\"},{\"archive_size_bytes\":221521137,\"id\":\"113\",\"kind\":\"VM\",\"logical_id\":\"vm113\"},{\"archive_size_bytes\":10622796310,\"id\":\"130\",\"kind\":\"VM\",\"logical_id\":\"vm130\"},{\"archive_size_bytes\":890940072,\"id\":\"150\",\"kind\":\"VM\",\"logical_id\":\"vm150\"},{\"archive_size_bytes\":3469023634,\"id\":\"160\",\"kind\":\"VM\",\"logical_id\":\"vm160\"},{\"archive_size_bytes\":7824247513,\"id\":\"170\",\"kind\":\"VM\",\"logical_id\":\"vm170\"},{\"archive_size_bytes\":2026787893,\"id\":\"171\",\"kind\":\"VM\",\"logical_id\":\"vm171\"},{\"archive_size_bytes\":1176306536,\"id\":\"180\",\"kind\":\"VM\",\"logical_id\":\"vm180\"},{\"archive_size_bytes\":1551688736,\"id\":\"190\",\"kind\":\"VM\",\"logical_id\":\"vm190\"},{\"archive_size_bytes\":11005893463,\"id\":\"9130\",\"kind\":\"VM\",\"logical_id\":\"vm9130\"},{\"archive_size_bytes\":383573170,\"id\":\"200\",\"kind\":\"VM\",\"logical_id\":\"vm200\"},{\"archive_size_bytes\":3193,\"id\":\"filebrowser\",\"kind\":\"APP\",\"logical_id\":\"app-filebrowser\"},{\"archive_size_bytes\":152745,\"id\":\"skladchik-reports-monitor\",\"kind\":\"APP\",\"logical_id\":\"app-skladchik-reports-monitor\"},{\"archive_size_bytes\":270193536,\"id\":\"immich\",\"kind\":\"APP\",\"logical_id\":\"app-immich\"},{\"archive_size_bytes\":75298798,\"id\":\"hbarhub\",\"kind\":\"APP\",\"logical_id\":\"app-hbarhub\"},{\"archive_size_bytes\":8959200,\"id\":\"control-bootstrap-v2\",\"kind\":\"CONTROL\",\"logical_id\":\"control-bootstrap-v2\"}]}\n{\"clouds\":[{\"free_bytes\":792267937635,\"remote\":\"pve01-mail-01-crypt\",\"total_bytes\":1099511627776,\"used_bytes\":307243690141},{\"free_bytes\":792267873776,\"remote\":\"pve01-mail-02-crypt\",\"total_bytes\":1099511627776,\"used_bytes\":307243754000}],\"observed_at_epoch\":1788447976,\"schema\":\"homelab-backup-observability-snapshot-v1\",\"targets\":[{\"archive_size_bytes\":474143241,\"id\":\"110\",\"kind\":\"VM\",\"logical_id\":\"vm110\"},{\"archive_size_bytes\":369191552,\"id\":\"111\",\"kind\":\"VM\",\"logical_id\":\"vm111\"},{\"archive_size_bytes\":223627414,\"id\":\"112\",\"kind\":\"VM\",\"logical_id\":\"vm112\"},{\"archive_size_bytes\":221521137,\"id\":\"113\",\"kind\":\"VM\",\"logical_id\":\"vm113\"},{\"archive_size_bytes\":10622796310,\"id\":\"130\",\"kind\":\"VM\",\"logical_id\":\"vm130\"},{\"archive_size_bytes\":890940072,\"id\":\"150\",\"kind\":\"VM\",\"logical_id\":\"vm150\"},{\"archive_size_bytes\":3469023634,\"id\":\"160\",\"kind\":\"VM\",\"logical_id\":\"vm160\"},{\"archive_size_bytes\":7824247513,\"id\":\"170\",\"kind\":\"VM\",\"logical_id\":\"vm170\"},{\"archive_size_bytes\":2026787893,\"id\":\"171\",\"kind\":\"VM\",\"logical_id\":\"vm171\"},{\"archive_size_bytes\":1176306536,\"id\":\"180\",\"kind\":\"VM\",\"logical_id\":\"vm180\"},{\"archive_size_bytes\":1551688736,\"id\":\"190\",\"kind\":\"VM\",\"logical_id\":\"vm190\"},{\"archive_size_bytes\":11005893463,\"id\":\"9130\",\"kind\":\"VM\",\"logical_id\":\"vm9130\"},{\"archive_size_bytes\":383573170,\"id\":\"200\",\"kind\":\"VM\",\"logical_id\":\"vm200\"},{\"archive_size_bytes\":3193,\"id\":\"filebrowser\",\"kind\":\"APP\",\"logical_id\":\"app-filebrowser\"},{\"archive_size_bytes\":152745,\"id\":\"skladchik-reports-monitor\",\"kind\":\"APP\",\"logical_id\":\"app-skladchik-reports-monitor\"},{\"archive_size_bytes\":270193536,\"id\":\"immich\",\"kind\":\"APP\",\"logical_id\":\"app-immich\"},{\"archive_size_bytes\":75298798,\"id\":\"hbarhub\",\"kind\":\"APP\",\"logical_id\":\"app-hbarhub\"},{\"archive_size_bytes\":8959200,\"id\":\"control-bootstrap-v2\",\"kind\":\"CONTROL\",\"logical_id\":\"control-bootstrap-v2\"}]}\n{\"clouds\":[{\"free_bytes\":792267937635,\"remote\":\"pve01-mail-01-crypt\",\"total_bytes\":1099511627776,\"used_bytes\":307243690141},{\"free_bytes\":792267873776,\"remote\":\"pve01-mail-02-crypt\",\"total_bytes\":1099511627776,\"used_bytes\":307243754000}],\"observed_at_epoch\":1788451586,\"schema\":\"homelab-backup-observability-snapshot-v1\",\"targets\":[{\"archive_size_bytes\":474143241,\"id\":\"110\",\"kind\":\"VM\",\"logical_id\":\"vm110\"},{\"archive_size_bytes\":369191552,\"id\":\"111\",\"kind\":\"VM\",\"logical_id\":\"vm111\"},{\"archive_size_bytes\":223627414,\"id\":\"112\",\"kind\":\"VM\",\"logical_id\":\"vm112\"},{\"archive_size_bytes\":221521137,\"id\":\"113\",\"kind\":\"VM\",\"logical_id\":\"vm113\"},{\"archive_size_bytes\":10622796310,\"id\":\"130\",\"kind\":\"VM\",\"logical_id\":\"vm130\"},{\"archive_size_bytes\":890940072,\"id\":\"150\",\"kind\":\"VM\",\"logical_id\":\"vm150\"},{\"archive_size_bytes\":3469023634,\"id\":\"160\",\"kind\":\"VM\",\"logical_id\":\"vm160\"},{\"archive_size_bytes\":7824247513,\"id\":\"170\",\"kind\":\"VM\",\"logical_id\":\"vm170\"},{\"archive_size_bytes\":2026787893,\"id\":\"171\",\"kind\":\"VM\",\"logical_id\":\"vm171\"},{\"archive_size_bytes\":1176306536,\"id\":\"180\",\"kind\":\"VM\",\"logical_id\":\"vm180\"},{\"archive_size_bytes\":1551688736,\"id\":\"190\",\"kind\":\"VM\",\"logical_id\":\"vm190\"},{\"archive_size_bytes\":11005893463,\"id\":\"9130\",\"kind\":\"VM\",\"logical_id\":\"vm9130\"},{\"archive_size_bytes\":383573170,\"id\":\"200\",\"kind\":\"VM\",\"logical_id\":\"vm200\"},{\"archive_size_bytes\":3193,\"id\":\"filebrowser\",\"kind\":\"APP\",\"logical_id\":\"app-filebrowser\"},{\"archive_size_bytes\":152745,\"id\":\"skladchik-reports-monitor\",\"kind\":\"APP\",\"logical_id\":\"app-skladchik-reports-monitor\"},{\"archive_size_bytes\":270193536,\"id\":\"immich\",\"kind\":\"APP\",\"logical_id\":\"app-immich\"},{\"archive_size_bytes\":75298798,\"id\":\"hbarhub\",\"kind\":\"APP\",\"logical_id\":\"app-hbarhub\"},{\"archive_size_bytes\":8959200,\"id\":\"control-bootstrap-v2\",\"kind\":\"CONTROL\",\"logical_id\":\"control-bootstrap-v2\"}]}\n{\"clouds\":[{\"free_bytes\":771612988784,\"remote\":\"pve01-mail-01-crypt\",\"total_bytes\":1099511627776,\"used_bytes\":327898638992},{\"free_bytes\":771612924925,\"remote\":\"pve01-mail-02-crypt\",\"total_bytes\":1099511627776,\"used_bytes\":327898702851}],\"observed_at_epoch\":1788456981,\"schema\":\"homelab-backup-observability-snapshot-v1\",\"targets\":[{\"archive_size_bytes\":476345840,\"id\":\"110\",\"kind\":\"VM\",\"logical_id\":\"vm110\"},{\"archive_size_bytes\":369191552,\"id\":\"111\",\"kind\":\"VM\",\"logical_id\":\"vm111\"},{\"archive_size_bytes\":223627414,\"id\":\"112\",\"kind\":\"VM\",\"logical_id\":\"vm112\"},{\"archive_size_bytes\":221521137,\"id\":\"113\",\"kind\":\"VM\",\"logical_id\":\"vm113\"},{\"archive_size_bytes\":892033158,\"id\":\"150\",\"kind\":\"VM\",\"logical_id\":\"vm150\"},{\"archive_size_bytes\":3469023634,\"id\":\"160\",\"kind\":\"VM\",\"logical_id\":\"vm160\"},{\"archive_size_bytes\":7937058725,\"id\":\"170\",\"kind\":\"VM\",\"logical_id\":\"vm170\"},{\"archive_size_bytes\":2026787893,\"id\":\"171\",\"kind\":\"VM\",\"logical_id\":\"vm171\"},{\"archive_size_bytes\":1176306536,\"id\":\"180\",\"kind\":\"VM\",\"logical_id\":\"vm180\"},{\"archive_size_bytes\":1551688736,\"id\":\"190\",\"kind\":\"VM\",\"logical_id\":\"vm190\"},{\"archive_size_bytes\":11005893463,\"id\":\"9130\",\"kind\":\"VM\",\"logical_id\":\"vm9130\"},{\"archive_size_bytes\":386825665,\"id\":\"200\",\"kind\":\"VM\",\"logical_id\":\"vm200\"},{\"archive_size_bytes\":3192,\"id\":\"filebrowser\",\"kind\":\"APP\",\"logical_id\":\"app-filebrowser\"},{\"archive_size_bytes\":153389,\"id\":\"skladchik-reports-monitor\",\"kind\":\"APP\",\"logical_id\":\"app-skladchik-reports-monitor\"},{\"archive_size_bytes\":270193550,\"id\":\"immich\",\"kind\":\"APP\",\"logical_id\":\"app-immich\"},{\"archive_size_bytes\":75379068,\"id\":\"hbarhub\",\"kind\":\"APP\",\"logical_id\":\"app-hbarhub\"},{\"archive_size_bytes\":8959200,\"id\":\"control-bootstrap-v2\",\"kind\":\"CONTROL\",\"logical_id\":\"control-bootstrap-v2\"}]}\n{\"clouds\":[{\"free_bytes\":755855450788,\"remote\":\"pve01-mail-01-crypt\",\"total_bytes\":1099511627776,\"used_bytes\":343656176988},{\"free_bytes\":755855386929,\"remote\":\"pve01-mail-02-crypt\",\"total_bytes\":1099511627776,\"used_bytes\":343656240847}],\"observed_at_epoch\":1788460613,\"schema\":\"homelab-backup-observability-snapshot-v1\",\"targets\":[{\"archive_size_bytes\":476345840,\"id\":\"110\",\"kind\":\"VM\",\"logical_id\":\"vm110\"},{\"archive_size_bytes\":369191552,\"id\":\"111\",\"kind\":\"VM\",\"logical_id\":\"vm111\"},{\"archive_size_bytes\":223627414,\"id\":\"112\",\"kind\":\"VM\",\"logical_id\":\"vm112\"},{\"archive_size_bytes\":221521137,\"id\":\"113\",\"kind\":\"VM\",\"logical_id\":\"vm113\"},{\"archive_size_bytes\":10611899437,\"id\":\"130\",\"kind\":\"VM\",\"logical_id\":\"vm130\"},{\"archive_size_bytes\":892033158,\"id\":\"150\",\"kind\":\"VM\",\"logical_id\":\"vm150\"},{\"archive_size_bytes\":3469023634,\"id\":\"160\",\"kind\":\"VM\",\"logical_id\":\"vm160\"},{\"archive_size_bytes\":7937058725,\"id\":\"170\",\"kind\":\"VM\",\"logical_id\":\"vm170\"},{\"archive_size_bytes\":2026047659,\"id\":\"171\",\"kind\":\"VM\",\"logical_id\":\"vm171\"},{\"archive_size_bytes\":1171789442,\"id\":\"180\",\"kind\":\"VM\",\"logical_id\":\"vm180\"},{\"archive_size_bytes\":1557482946,\"id\":\"190\",\"kind\":\"VM\",\"logical_id\":\"vm190\"},{\"archive_size_bytes\":386825665,\"id\":\"200\",\"kind\":\"VM\",\"logical_id\":\"vm200\"},{\"archive_size_bytes\":3192,\"id\":\"filebrowser\",\"kind\":\"APP\",\"logical_id\":\"app-filebrowser\"},{\"archive_size_bytes\":153389,\"id\":\"skladchik-reports-monitor\",\"kind\":\"APP\",\"logical_id\":\"app-skladchik-reports-monitor\"},{\"archive_size_bytes\":270193550,\"id\":\"immich\",\"kind\":\"APP\",\"logical_id\":\"app-immich\"},{\"archive_size_bytes\":75379068,\"id\":\"hbarhub\",\"kind\":\"APP\",\"logical_id\":\"app-hbarhub\"},{\"archive_size_bytes\":8959200,\"id\":\"control-bootstrap-v2\",\"kind\":\"CONTROL\",\"logical_id\":\"control-bootstrap-v2\"}]}\n{\"clouds\":[{\"free_bytes\":751486275845,\"remote\":\"pve01-mail-01-crypt\",\"total_bytes\":1099511627776,\"used_bytes\":348025351931},{\"free_bytes\":751486211986,\"remote\":\"pve01-mail-02-crypt\",\"total_bytes\":1099511627776,\"used_bytes\":348025415790}],\"observed_at_epoch\":1788465932,\"schema\":\"homelab-backup-observability-snapshot-v1\",\"targets\":[{\"archive_size_bytes\":476345840,\"id\":\"110\",\"kind\":\"VM\",\"logical_id\":\"vm110\"},{\"archive_size_bytes\":369892675,\"id\":\"111\",\"kind\":\"VM\",\"logical_id\":\"vm111\"},{\"archive_size_bytes\":223672749,\"id\":\"112\",\"kind\":\"VM\",\"logical_id\":\"vm112\"},{\"archive_size_bytes\":221529138,\"id\":\"113\",\"kind\":\"VM\",\"logical_id\":\"vm113\"},{\"archive_size_bytes\":10611899437,\"id\":\"130\",\"kind\":\"VM\",\"logical_id\":\"vm130\"},{\"archive_size_bytes\":892033158,\"id\":\"150\",\"kind\":\"VM\",\"logical_id\":\"vm150\"},{\"archive_size_bytes\":3553006761,\"id\":\"160\",\"kind\":\"VM\",\"logical_id\":\"vm160\"},{\"archive_size_bytes\":7937058725,\"id\":\"170\",\"kind\":\"VM\",\"logical_id\":\"vm170\"},{\"archive_size_bytes\":2026047659,\"id\":\"171\",\"kind\":\"VM\",\"logical_id\":\"vm171\"},{\"archive_size_bytes\":1171789442,\"id\":\"180\",\"kind\":\"VM\",\"logical_id\":\"vm180\"},{\"archive_size_bytes\":1557482946,\"id\":\"190\",\"kind\":\"VM\",\"logical_id\":\"vm190\"},{\"archive_size_bytes\":10998362942,\"id\":\"9130\",\"kind\":\"VM\",\"logical_id\":\"vm9130\"},{\"archive_size_bytes\":386825665,\"id\":\"200\",\"kind\":\"VM\",\"logical_id\":\"vm200\"},{\"archive_size_bytes\":3192,\"id\":\"filebrowser\",\"kind\":\"APP\",\"logical_id\":\"app-filebrowser\"},{\"archive_size_bytes\":153389,\"id\":\"skladchik-reports-monitor\",\"kind\":\"APP\",\"logical_id\":\"app-skladchik-reports-monitor\"},{\"archive_size_bytes\":270193550,\"id\":\"immich\",\"kind\":\"APP\",\"logical_id\":\"app-immich\"},{\"archive_size_bytes\":75379068,\"id\":\"hbarhub\",\"kind\":\"APP\",\"logical_id\":\"app-hbarhub\"},{\"archive_size_bytes\":8959200,\"id\":\"control-bootstrap-v2\",\"kind\":\"CONTROL\",\"logical_id\":\"control-bootstrap-v2\"}]}\n{\"clouds\":[{\"free_bytes\":751477313023,\"remote\":\"pve01-mail-01-crypt\",\"total_bytes\":1099511627776,\"used_bytes\":348034314753},{\"free_bytes\":751477249164,\"remote\":\"pve01-mail-02-crypt\",\"total_bytes\":1099511627776,\"used_bytes\":348034378612}],\"observed_at_epoch\":1788469583,\"schema\":\"homelab-backup-observability-snapshot-v1\",\"targets\":[{\"archive_size_bytes\":476345840,\"id\":\"110\",\"kind\":\"VM\",\"logical_id\":\"vm110\"},{\"archive_size_bytes\":369892675,\"id\":\"111\",\"kind\":\"VM\",\"logical_id\":\"vm111\"},{\"archive_size_bytes\":223672749,\"id\":\"112\",\"kind\":\"VM\",\"logical_id\":\"vm112\"},{\"archive_size_bytes\":221529138,\"id\":\"113\",\"kind\":\"VM\",\"logical_id\":\"vm113\"},{\"archive_size_bytes\":10611899437,\"id\":\"130\",\"kind\":\"VM\",\"logical_id\":\"vm130\"},{\"archive_size_bytes\":892033158,\"id\":\"150\",\"kind\":\"VM\",\"logical_id\":\"vm150\"},{\"archive_size_bytes\":3553006761,\"id\":\"160\",\"kind\":\"VM\",\"logical_id\":\"vm160\"},{\"archive_size_bytes\":7937058725,\"id\":\"170\",\"kind\":\"VM\",\"logical_id\":\"vm170\"},{\"archive_size_bytes\":2026047659,\"id\":\"171\",\"kind\":\"VM\",\"logical_id\":\"vm171\"},{\"archive_size_bytes\":1171789442,\"id\":\"180\",\"kind\":\"VM\",\"logical_id\":\"vm180\"},{\"archive_size_bytes\":1557482946,\"id\":\"190\",\"kind\":\"VM\",\"logical_id\":\"vm190\"},{\"archive_size_bytes\":10998362942,\"id\":\"9130\",\"kind\":\"VM\",\"logical_id\":\"vm9130\"},{\"archive_size_bytes\":386825665,\"id\":\"200\",\"kind\":\"VM\",\"logical_id\":\"vm200\"},{\"archive_size_bytes\":3192,\"id\":\"filebrowser\",\"kind\":\"APP\",\"logical_id\":\"app-filebrowser\"},{\"archive_size_bytes\":153389,\"id\":\"skladchik-reports-monitor\",\"kind\":\"APP\",\"logical_id\":\"app-skladchik-reports-monitor\"},{\"archive_size_bytes\":270193550,\"id\":\"immich\",\"kind\":\"APP\",\"logical_id\":\"app-immich\"},{\"archive_size_bytes\":75379068,\"id\":\"hbarhub\",\"kind\":\"APP\",\"logical_id\":\"app-hbarhub\"},{\"archive_size_bytes\":8959291,\"id\":\"control-bootstrap-v2\",\"kind\":\"CONTROL\",\"logical_id\":\"control-bootstrap-v2\"}]}\n{\"clouds\":[{\"free_bytes\":751477313023,\"remote\":\"pve01-mail-01-crypt\",\"total_bytes\":1099511627776,\"used_bytes\":348034314753},{\"free_bytes\":751477249164,\"remote\":\"pve01-mail-02-crypt\",\"total_bytes\":1099511627776,\"used_bytes\":348034378612}],\"observed_at_epoch\":1788474989,\"schema\":\"homelab-backup-observability-snapshot-v1\",\"targets\":[{\"archive_size_bytes\":476345840,\"id\":\"110\",\"kind\":\"VM\",\"logical_id\":\"vm110\"},{\"archive_size_bytes\":369892675,\"id\":\"111\",\"kind\":\"VM\",\"logical_id\":\"vm111\"},{\"archive_size_bytes\":223672749,\"id\":\"112\",\"kind\":\"VM\",\"logical_id\":\"vm112\"},{\"archive_size_bytes\":221529138,\"id\":\"113\",\"kind\":\"VM\",\"logical_id\":\"vm113\"},{\"archive_size_bytes\":10611899437,\"id\":\"130\",\"kind\":\"VM\",\"logical_id\":\"vm130\"},{\"archive_size_bytes\":892033158,\"id\":\"150\",\"kind\":\"VM\",\"logical_id\":\"vm150\"},{\"archive_size_bytes\":3553006761,\"id\":\"160\",\"kind\":\"VM\",\"logical_id\":\"vm160\"},{\"archive_size_bytes\":7937058725,\"id\":\"170\",\"kind\":\"VM\",\"logical_id\":\"vm170\"},{\"archive_size_bytes\":2026047659,\"id\":\"171\",\"kind\":\"VM\",\"logical_id\":\"vm171\"},{\"archive_size_bytes\":1171789442,\"id\":\"180\",\"kind\":\"VM\",\"logical_id\":\"vm180\"},{\"archive_size_bytes\":1557482946,\"id\":\"190\",\"kind\":\"VM\",\"logical_id\":\"vm190\"},{\"archive_size_bytes\":10998362942,\"id\":\"9130\",\"kind\":\"VM\",\"logical_id\":\"vm9130\"},{\"archive_size_bytes\":386825665,\"id\":\"200\",\"kind\":\"VM\",\"logical_id\":\"vm200\"},{\"archive_size_bytes\":3192,\"id\":\"filebrowser\",\"kind\":\"APP\",\"logical_id\":\"app-filebrowser\"},{\"archive_size_bytes\":153389,\"id\":\"skladchik-reports-monitor\",\"kind\":\"APP\",\"logical_id\":\"app-skladchik-reports-monitor\"},{\"archive_size_bytes\":270193550,\"id\":\"immich\",\"kind\":\"APP\",\"logical_id\":\"app-immich\"},{\"archive_size_bytes\":75379068,\"id\":\"hbarhub\",\"kind\":\"APP\",\"logical_id\":\"app-hbarhub\"},{\"archive_size_bytes\":8959291,\"id\":\"control-bootstrap-v2\",\"kind\":\"CONTROL\",\"logical_id\":\"control-bootstrap-v2\"}]}\n{\"clouds\":[{\"free_bytes\":751477313023,\"remote\":\"pve01-mail-01-crypt\",\"total_bytes\":1099511627776,\"used_bytes\":348034314753},{\"free_bytes\":751477249164,\"remote\":\"pve01-mail-02-crypt\",\"total_bytes\":1099511627776,\"used_bytes\":348034378612}],\"observed_at_epoch\":1788478603,\"schema\":\"homelab-backup-observability-snapshot-v1\",\"targets\":[{\"archive_size_bytes\":476345840,\"id\":\"110\",\"kind\":\"VM\",\"logical_id\":\"vm110\"},{\"archive_size_bytes\":369892675,\"id\":\"111\",\"kind\":\"VM\",\"logical_id\":\"vm111\"},{\"archive_size_bytes\":223672749,\"id\":\"112\",\"kind\":\"VM\",\"logical_id\":\"vm112\"},{\"archive_size_bytes\":221529138,\"id\":\"113\",\"kind\":\"VM\",\"logical_id\":\"vm113\"},{\"archive_size_bytes\":10611899437,\"id\":\"130\",\"kind\":\"VM\",\"logical_id\":\"vm130\"},{\"archive_size_bytes\":892033158,\"id\":\"150\",\"kind\":\"VM\",\"logical_id\":\"vm150\"},{\"archive_size_bytes\":3553006761,\"id\":\"160\",\"kind\":\"VM\",\"logical_id\":\"vm160\"},{\"archive_size_bytes\":7937058725,\"id\":\"170\",\"kind\":\"VM\",\"logical_id\":\"vm170\"},{\"archive_size_bytes\":2026047659,\"id\":\"171\",\"kind\":\"VM\",\"logical_id\":\"vm171\"},{\"archive_size_bytes\":1171789442,\"id\":\"180\",\"kind\":\"VM\",\"logical_id\":\"vm180\"},{\"archive_size_bytes\":1557482946,\"id\":\"190\",\"kind\":\"VM\",\"logical_id\":\"vm190\"},{\"archive_size_bytes\":10998362942,\"id\":\"9130\",\"kind\":\"VM\",\"logical_id\":\"vm9130\"},{\"archive_size_bytes\":386825665,\"id\":\"200\",\"kind\":\"VM\",\"logical_id\":\"vm200\"},{\"archive_size_bytes\":3192,\"id\":\"filebrowser\",\"kind\":\"APP\",\"logical_id\":\"app-filebrowser\"},{\"archive_size_bytes\":153389,\"id\":\"skladchik-reports-monitor\",\"kind\":\"APP\",\"logical_id\":\"app-skladchik-reports-monitor\"},{\"archive_size_bytes\":270193550,\"id\":\"immich\",\"kind\":\"APP\",\"logical_id\":\"app-immich\"},{\"archive_size_bytes\":75379068,\"id\":\"hbarhub\",\"kind\":\"APP\",\"logical_id\":\"app-hbarhub\"},{\"archive_size_bytes\":8959291,\"id\":\"control-bootstrap-v2\",\"kind\":\"CONTROL\",\"logical_id\":\"control-bootstrap-v2\"}]}\n{\"clouds\":[{\"free_bytes\":751477313023,\"remote\":\"pve01-mail-01-crypt\",\"total_bytes\":1099511627776,\"used_bytes\":348034314753},{\"free_bytes\":751477249164,\"remote\":\"pve01-mail-02-crypt\",\"total_bytes\":1099511627776,\"used_bytes\":348034378612}],\"observed_at_epoch\":1788482209,\"schema\":\"homelab-backup-observability-snapshot-v1\",\"targets\":[{\"archive_size_bytes\":476345840,\"id\":\"110\",\"kind\":\"VM\",\"logical_id\":\"vm110\"},{\"archive_size_bytes\":369892675,\"id\":\"111\",\"kind\":\"VM\",\"logical_id\":\"vm111\"},{\"archive_size_bytes\":223672749,\"id\":\"112\",\"kind\":\"VM\",\"logical_id\":\"vm112\"},{\"archive_size_bytes\":221529138,\"id\":\"113\",\"kind\":\"VM\",\"logical_id\":\"vm113\"},{\"archive_size_bytes\":10611899437,\"id\":\"130\",\"kind\":\"VM\",\"logical_id\":\"vm130\"},{\"archive_size_bytes\":892033158,\"id\":\"150\",\"kind\":\"VM\",\"logical_id\":\"vm150\"},{\"archive_size_bytes\":3553006761,\"id\":\"160\",\"kind\":\"VM\",\"logical_id\":\"vm160\"},{\"archive_size_bytes\":7937058725,\"id\":\"170\",\"kind\":\"VM\",\"logical_id\":\"vm170\"},{\"archive_size_bytes\":2026047659,\"id\":\"171\",\"kind\":\"VM\",\"logical_id\":\"vm171\"},{\"archive_size_bytes\":1171789442,\"id\":\"180\",\"kind\":\"VM\",\"logical_id\":\"vm180\"},{\"archive_size_bytes\":1557482946,\"id\":\"190\",\"kind\":\"VM\",\"logical_id\":\"vm190\"},{\"archive_size_bytes\":10998362942,\"id\":\"9130\",\"kind\":\"VM\",\"logical_id\":\"vm9130\"},{\"archive_size_bytes\":386825665,\"id\":\"200\",\"kind\":\"VM\",\"logical_id\":\"vm200\"},{\"archive_size_bytes\":3192,\"id\":\"filebrowser\",\"kind\":\"APP\",\"logical_id\":\"app-filebrowser\"},{\"archive_size_bytes\":153389,\"id\":\"skladchik-reports-monitor\",\"kind\":\"APP\",\"logical_id\":\"app-skladchik-reports-monitor\"},{\"archive_size_bytes\":270193550,\"id\":\"immich\",\"kind\":\"APP\",\"logical_id\":\"app-immich\"},{\"archive_size_bytes\":75379068,\"id\":\"hbarhub\",\"kind\":\"APP\",\"logical_id\":\"app-hbarhub\"},{\"archive_size_bytes\":8959291,\"id\":\"control-bootstrap-v2\",\"kind\":\"CONTROL\",\"logical_id\":\"control-bootstrap-v2\"}]}\n{\"clouds\":[{\"free_bytes\":751477313023,\"remote\":\"pve01-mail-01-crypt\",\"total_bytes\":1099511627776,\"used_bytes\":348034314753},{\"free_bytes\":751477249164,\"remote\":\"pve01-mail-02-crypt\",\"total_bytes\":1099511627776,\"used_bytes\":348034378612}],\"observed_at_epoch\":1788487523,\"schema\":\"homelab-backup-observability-snapshot-v1\",\"targets\":[{\"archive_size_bytes\":476345840,\"id\":\"110\",\"kind\":\"VM\",\"logical_id\":\"vm110\"},{\"archive_size_bytes\":369892675,\"id\":\"111\",\"kind\":\"VM\",\"logical_id\":\"vm111\"},{\"archive_size_bytes\":223672749,\"id\":\"112\",\"kind\":\"VM\",\"logical_id\":\"vm112\"},{\"archive_size_bytes\":221529138,\"id\":\"113\",\"kind\":\"VM\",\"logical_id\":\"vm113\"},{\"archive_size_bytes\":10611899437,\"id\":\"130\",\"kind\":\"VM\",\"logical_id\":\"vm130\"},{\"archive_size_bytes\":892033158,\"id\":\"150\",\"kind\":\"VM\",\"logical_id\":\"vm150\"},{\"archive_size_bytes\":3553006761,\"id\":\"160\",\"kind\":\"VM\",\"logical_id\":\"vm160\"},{\"archive_size_bytes\":7937058725,\"id\":\"170\",\"kind\":\"VM\",\"logical_id\":\"vm170\"},{\"archive_size_bytes\":2026047659,\"id\":\"171\",\"kind\":\"VM\",\"logical_id\":\"vm171\"},{\"archive_size_bytes\":1171789442,\"id\":\"180\",\"kind\":\"VM\",\"logical_id\":\"vm180\"},{\"archive_size_bytes\":1557482946,\"id\":\"190\",\"kind\":\"VM\",\"logical_id\":\"vm190\"},{\"archive_size_bytes\":10998362942,\"id\":\"9130\",\"kind\":\"VM\",\"logical_id\":\"vm9130\"},{\"archive_size_bytes\":386825665,\"id\":\"200\",\"kind\":\"VM\",\"logical_id\":\"vm200\"},{\"archive_size_bytes\":3192,\"id\":\"filebrowser\",\"kind\":\"APP\",\"logical_id\":\"app-filebrowser\"},{\"archive_size_bytes\":153389,\"id\":\"skladchik-reports-monitor\",\"kind\":\"APP\",\"logical_id\":\"app-skladchik-reports-monitor\"},{\"archive_size_bytes\":270193550,\"id\":\"immich\",\"kind\":\"APP\",\"logical_id\":\"app-immich\"},{\"archive_size_bytes\":75379068,\"id\":\"hbarhub\",\"kind\":\"APP\",\"logical_id\":\"app-hbarhub\"},{\"archive_size_bytes\":8959291,\"id\":\"control-bootstrap-v2\",\"kind\":\"CONTROL\",\"logical_id\":\"control-bootstrap-v2\"}]}\n{\"clouds\":[{\"free_bytes\":751477313023,\"remote\":\"pve01-mail-01-crypt\",\"total_bytes\":1099511627776,\"used_bytes\":348034314753},{\"free_bytes\":751477249164,\"remote\":\"pve01-mail-02-crypt\",\"total_bytes\":1099511627776,\"used_bytes\":348034378612}],\"observed_at_epoch\":1788491188,\"schema\":\"homelab-backup-observability-snapshot-v1\",\"targets\":[{\"archive_size_bytes\":476345840,\"id\":\"110\",\"kind\":\"VM\",\"logical_id\":\"vm110\"},{\"archive_size_bytes\":369892675,\"id\":\"111\",\"kind\":\"VM\",\"logical_id\":\"vm111\"},{\"archive_size_bytes\":223672749,\"id\":\"112\",\"kind\":\"VM\",\"logical_id\":\"vm112\"},{\"archive_size_bytes\":221529138,\"id\":\"113\",\"kind\":\"VM\",\"logical_id\":\"vm113\"},{\"archive_size_bytes\":10611899437,\"id\":\"130\",\"kind\":\"VM\",\"logical_id\":\"vm130\"},{\"archive_size_bytes\":892033158,\"id\":\"150\",\"kind\":\"VM\",\"logical_id\":\"vm150\"},{\"archive_size_bytes\":3553006761,\"id\":\"160\",\"kind\":\"VM\",\"logical_id\":\"vm160\"},{\"archive_size_bytes\":7937058725,\"id\":\"170\",\"kind\":\"VM\",\"logical_id\":\"vm170\"},{\"archive_size_bytes\":2026047659,\"id\":\"171\",\"kind\":\"VM\",\"logical_id\":\"vm171\"},{\"archive_size_bytes\":1171789442,\"id\":\"180\",\"kind\":\"VM\",\"logical_id\":\"vm180\"},{\"archive_size_bytes\":1557482946,\"id\":\"190\",\"kind\":\"VM\",\"logical_id\":\"vm190\"},{\"archive_size_bytes\":10998362942,\"id\":\"9130\",\"kind\":\"VM\",\"logical_id\":\"vm9130\"},{\"archive_size_bytes\":386825665,\"id\":\"200\",\"kind\":\"VM\",\"logical_id\":\"vm200\"},{\"archive_size_bytes\":3192,\"id\":\"filebrowser\",\"kind\":\"APP\",\"logical_id\":\"app-filebrowser\"},{\"archive_size_bytes\":153389,\"id\":\"skladchik-reports-monitor\",\"kind\":\"APP\",\"logical_id\":\"app-skladchik-reports-monitor\"},{\"archive_size_bytes\":270193550,\"id\":\"immich\",\"kind\":\"APP\",\"logical_id\":\"app-immich\"},{\"archive_size_bytes\":75379068,\"id\":\"hbarhub\",\"kind\":\"APP\",\"logical_id\":\"app-hbarhub\"},{\"archive_size_bytes\":8959291,\"id\":\"control-bootstrap-v2\",\"kind\":\"CONTROL\",\"logical_id\":\"control-bootstrap-v2\"}]}\n{\"clouds\":[{\"free_bytes\":835463704954,\"remote\":\"pve01-mail-01-crypt\",\"total_bytes\":1099511627776,\"used_bytes\":264047922822},{\"free_bytes\":835463641095,\"remote\":\"pve01-mail-02-crypt\",\"total_bytes\":1099511627776,\"used_bytes\":264047986681}],\"observed_at_epoch\":1788496558,\"schema\":\"homelab-backup-observability-snapshot-v1\",\"targets\":[{\"archive_size_bytes\":476345840,\"id\":\"110\",\"kind\":\"VM\",\"logical_id\":\"vm110\"},{\"archive_size_bytes\":369892675,\"id\":\"111\",\"kind\":\"VM\",\"logical_id\":\"vm111\"},{\"archive_size_bytes\":223672749,\"id\":\"112\",\"kind\":\"VM\",\"logical_id\":\"vm112\"},{\"archive_size_bytes\":221529138,\"id\":\"113\",\"kind\":\"VM\",\"logical_id\":\"vm113\"},{\"archive_size_bytes\":10611899437,\"id\":\"130\",\"kind\":\"VM\",\"logical_id\":\"vm130\"},{\"archive_size_bytes\":892033158,\"id\":\"150\",\"kind\":\"VM\",\"logical_id\":\"vm150\"},{\"archive_size_bytes\":3553006761,\"id\":\"160\",\"kind\":\"VM\",\"logical_id\":\"vm160\"},{\"archive_size_bytes\":7937058725,\"id\":\"170\",\"kind\":\"VM\",\"logical_id\":\"vm170\"},{\"archive_size_bytes\":2026047659,\"id\":\"171\",\"kind\":\"VM\",\"logical_id\":\"vm171\"},{\"archive_size_bytes\":1171789442,\"id\":\"180\",\"kind\":\"VM\",\"logical_id\":\"vm180\"},{\"archive_size_bytes\":1557482946,\"id\":\"190\",\"kind\":\"VM\",\"logical_id\":\"vm190\"},{\"archive_size_bytes\":10998362942,\"id\":\"9130\",\"kind\":\"VM\",\"logical_id\":\"vm9130\"},{\"archive_size_bytes\":386825665,\"id\":\"200\",\"kind\":\"VM\",\"logical_id\":\"vm200\"},{\"archive_size_bytes\":3192,\"id\":\"filebrowser\",\"kind\":\"APP\",\"logical_id\":\"app-filebrowser\"},{\"archive_size_bytes\":153389,\"id\":\"skladchik-reports-monitor\",\"kind\":\"APP\",\"logical_id\":\"app-skladchik-reports-monitor\"},{\"archive_size_bytes\":270193550,\"id\":\"immich\",\"kind\":\"APP\",\"logical_id\":\"app-immich\"},{\"archive_size_bytes\":75379068,\"id\":\"hbarhub\",\"kind\":\"APP\",\"logical_id\":\"app-hbarhub\"},{\"archive_size_bytes\":8959291,\"id\":\"control-bootstrap-v2\",\"kind\":\"CONTROL\",\"logical_id\":\"control-bootstrap-v2\"}]}\n{\"clouds\":[{\"free_bytes\":815114087404,\"remote\":\"pve01-mail-01-crypt\",\"total_bytes\":1099511627776,\"used_bytes\":284397540372},{\"free_bytes\":815114023545,\"remote\":\"pve01-mail-02-crypt\",\"total_bytes\":1099511627776,\"used_bytes\":284397604231}],\"observed_at_epoch\":1788503153,\"schema\":\"homelab-backup-observability-snapshot-v1\",\"targets\":[{\"archive_size_bytes\":478357303,\"id\":\"110\",\"kind\":\"VM\",\"logical_id\":\"vm110\"},{\"archive_size_bytes\":369892675,\"id\":\"111\",\"kind\":\"VM\",\"logical_id\":\"vm111\"},{\"archive_size_bytes\":223672749,\"id\":\"112\",\"kind\":\"VM\",\"logical_id\":\"vm112\"},{\"archive_size_bytes\":221529138,\"id\":\"113\",\"kind\":\"VM\",\"logical_id\":\"vm113\"},{\"archive_size_bytes\":887338816,\"id\":\"150\",\"kind\":\"VM\",\"logical_id\":\"vm150\"},{\"archive_size_bytes\":3553006761,\"id\":\"160\",\"kind\":\"VM\",\"logical_id\":\"vm160\"},{\"archive_size_bytes\":7974616162,\"id\":\"170\",\"kind\":\"VM\",\"logical_id\":\"vm170\"},{\"archive_size_bytes\":2026047659,\"id\":\"171\",\"kind\":\"VM\",\"logical_id\":\"vm171\"},{\"archive_size_bytes\":1557482946,\"id\":\"190\",\"kind\":\"VM\",\"logical_id\":\"vm190\"},{\"archive_size_bytes\":10998362942,\"id\":\"9130\",\"kind\":\"VM\",\"logical_id\":\"vm9130\"},{\"archive_size_bytes\":3192,\"id\":\"filebrowser\",\"kind\":\"APP\",\"logical_id\":\"app-filebrowser\"},{\"archive_size_bytes\":153389,\"id\":\"skladchik-reports-monitor\",\"kind\":\"APP\",\"logical_id\":\"app-skladchik-reports-monitor\"},{\"archive_size_bytes\":270193550,\"id\":\"immich\",\"kind\":\"APP\",\"logical_id\":\"app-immich\"},{\"archive_size_bytes\":75379068,\"id\":\"hbarhub\",\"kind\":\"APP\",\"logical_id\":\"app-hbarhub\"},{\"archive_size_bytes\":8959291,\"id\":\"control-bootstrap-v2\",\"kind\":\"CONTROL\",\"logical_id\":\"control-bootstrap-v2\"}]}\n{\"clouds\":[{\"free_bytes\":812739438805,\"remote\":\"pve01-mail-01-crypt\",\"total_bytes\":1099511627776,\"used_bytes\":286772188971},{\"free_bytes\":812739374946,\"remote\":\"pve01-mail-02-crypt\",\"total_bytes\":1099511627776,\"used_bytes\":286772252830}],\"observed_at_epoch\":1788507354,\"schema\":\"homelab-backup-observability-snapshot-v1\",\"targets\":[{\"archive_size_bytes\":478357303,\"id\":\"110\",\"kind\":\"VM\",\"logical_id\":\"vm110\"},{\"archive_size_bytes\":369892675,\"id\":\"111\",\"kind\":\"VM\",\"logical_id\":\"vm111\"},{\"archive_size_bytes\":223672749,\"id\":\"112\",\"kind\":\"VM\",\"logical_id\":\"vm112\"},{\"archive_size_bytes\":221529138,\"id\":\"113\",\"kind\":\"VM\",\"logical_id\":\"vm113\"},{\"archive_size_bytes\":10620176786,\"id\":\"130\",\"kind\":\"VM\",\"logical_id\":\"vm130\"},{\"archive_size_bytes\":887338816,\"id\":\"150\",\"kind\":\"VM\",\"logical_id\":\"vm150\"},{\"archive_size_bytes\":3553006761,\"id\":\"160\",\"kind\":\"VM\",\"logical_id\":\"vm160\"},{\"archive_size_bytes\":7974616162,\"id\":\"170\",\"kind\":\"VM\",\"logical_id\":\"vm170\"},{\"archive_size_bytes\":2028255985,\"id\":\"171\",\"kind\":\"VM\",\"logical_id\":\"vm171\"},{\"archive_size_bytes\":1557482946,\"id\":\"190\",\"kind\":\"VM\",\"logical_id\":\"vm190\"},{\"archive_size_bytes\":10998362942,\"id\":\"9130\",\"kind\":\"VM\",\"logical_id\":\"vm9130\"},{\"archive_size_bytes\":3197,\"id\":\"filebrowser\",\"kind\":\"APP\",\"logical_id\":\"app-filebrowser\"},{\"archive_size_bytes\":154716,\"id\":\"skladchik-reports-monitor\",\"kind\":\"APP\",\"logical_id\":\"app-skladchik-reports-monitor\"},{\"archive_size_bytes\":270193492,\"id\":\"immich\",\"kind\":\"APP\",\"logical_id\":\"app-immich\"},{\"archive_size_bytes\":75453412,\"id\":\"hbarhub\",\"kind\":\"APP\",\"logical_id\":\"app-hbarhub\"},{\"archive_size_bytes\":8959291,\"id\":\"control-bootstrap-v2\",\"kind\":\"CONTROL\",\"logical_id\":\"control-bootstrap-v2\"}]}\n{\"clouds\":[{\"free_bytes\":809167318295,\"remote\":\"pve01-mail-01-crypt\",\"total_bytes\":1099511627776,\"used_bytes\":290344309481},{\"free_bytes\":809167254436,\"remote\":\"pve01-mail-02-crypt\",\"total_bytes\":1099511627776,\"used_bytes\":290344373340}],\"observed_at_epoch\":1788512752,\"schema\":\"homelab-backup-observability-snapshot-v1\",\"targets\":[{\"archive_size_bytes\":478357303,\"id\":\"110\",\"kind\":\"VM\",\"logical_id\":\"vm110\"},{\"archive_size_bytes\":369892675,\"id\":\"111\",\"kind\":\"VM\",\"logical_id\":\"vm111\"},{\"archive_size_bytes\":223672749,\"id\":\"112\",\"kind\":\"VM\",\"logical_id\":\"vm112\"},{\"archive_size_bytes\":221529138,\"id\":\"113\",\"kind\":\"VM\",\"logical_id\":\"vm113\"},{\"archive_size_bytes\":10620176786,\"id\":\"130\",\"kind\":\"VM\",\"logical_id\":\"vm130\"},{\"archive_size_bytes\":887338816,\"id\":\"150\",\"kind\":\"VM\",\"logical_id\":\"vm150\"},{\"archive_size_bytes\":3571246511,\"id\":\"160\",\"kind\":\"VM\",\"logical_id\":\"vm160\"},{\"archive_size_bytes\":7974616162,\"id\":\"170\",\"kind\":\"VM\",\"logical_id\":\"vm170\"},{\"archive_size_bytes\":2028255985,\"id\":\"171\",\"kind\":\"VM\",\"logical_id\":\"vm171\"},{\"archive_size_bytes\":1557482946,\"id\":\"190\",\"kind\":\"VM\",\"logical_id\":\"vm190\"},{\"archive_size_bytes\":10998362942,\"id\":\"9130\",\"kind\":\"VM\",\"logical_id\":\"vm9130\"},{\"archive_size_bytes\":3197,\"id\":\"filebrowser\",\"kind\":\"APP\",\"logical_id\":\"app-filebrowser\"},{\"archive_size_bytes\":154716,\"id\":\"skladchik-reports-monitor\",\"kind\":\"APP\",\"logical_id\":\"app-skladchik-reports-monitor\"},{\"archive_size_bytes\":270193492,\"id\":\"immich\",\"kind\":\"APP\",\"logical_id\":\"app-immich\"},{\"archive_size_bytes\":75453412,\"id\":\"hbarhub\",\"kind\":\"APP\",\"logical_id\":\"app-hbarhub\"},{\"archive_size_bytes\":8959291,\"id\":\"control-bootstrap-v2\",\"kind\":\"CONTROL\",\"logical_id\":\"control-bootstrap-v2\"}]}\n{\"clouds\":[{\"free_bytes\":809158344080,\"remote\":\"pve01-mail-01-crypt\",\"total_bytes\":1099511627776,\"used_bytes\":290353283696},{\"free_bytes\":809158280221,\"remote\":\"pve01-mail-02-crypt\",\"total_bytes\":1099511627776,\"used_bytes\":290353347555}],\"observed_at_epoch\":1788516392,\"schema\":\"homelab-backup-observability-snapshot-v1\",\"targets\":[{\"archive_size_bytes\":478357303,\"id\":\"110\",\"kind\":\"VM\",\"logical_id\":\"vm110\"},{\"archive_size_bytes\":369892675,\"id\":\"111\",\"kind\":\"VM\",\"logical_id\":\"vm111\"},{\"archive_size_bytes\":223672749,\"id\":\"112\",\"kind\":\"VM\",\"logical_id\":\"vm112\"},{\"archive_size_bytes\":221529138,\"id\":\"113\",\"kind\":\"VM\",\"logical_id\":\"vm113\"},{\"archive_size_bytes\":10620176786,\"id\":\"130\",\"kind\":\"VM\",\"logical_id\":\"vm130\"},{\"archive_size_bytes\":887338816,\"id\":\"150\",\"kind\":\"VM\",\"logical_id\":\"vm150\"},{\"archive_size_bytes\":3571246511,\"id\":\"160\",\"kind\":\"VM\",\"logical_id\":\"vm160\"},{\"archive_size_bytes\":7974616162,\"id\":\"170\",\"kind\":\"VM\",\"logical_id\":\"vm170\"},{\"archive_size_bytes\":2028255985,\"id\":\"171\",\"kind\":\"VM\",\"logical_id\":\"vm171\"},{\"archive_size_bytes\":1557482946,\"id\":\"190\",\"kind\":\"VM\",\"logical_id\":\"vm190\"},{\"archive_size_bytes\":10998362942,\"id\":\"9130\",\"kind\":\"VM\",\"logical_id\":\"vm9130\"},{\"archive_size_bytes\":3197,\"id\":\"filebrowser\",\"kind\":\"APP\",\"logical_id\":\"app-filebrowser\"},{\"archive_size_bytes\":154716,\"id\":\"skladchik-reports-monitor\",\"kind\":\"APP\",\"logical_id\":\"app-skladchik-reports-monitor\"},{\"archive_size_bytes\":270193492,\"id\":\"immich\",\"kind\":\"APP\",\"logical_id\":\"app-immich\"},{\"archive_size_bytes\":75453412,\"id\":\"hbarhub\",\"kind\":\"APP\",\"logical_id\":\"app-hbarhub\"},{\"archive_size_bytes\":8970684,\"id\":\"control-bootstrap-v2\",\"kind\":\"CONTROL\",\"logical_id\":\"control-bootstrap-v2\"}]}\n{\"clouds\":[{\"free_bytes\":809158344080,\"remote\":\"pve01-mail-01-crypt\",\"total_bytes\":1099511627776,\"used_bytes\":290353283696},{\"free_bytes\":809158280221,\"remote\":\"pve01-mail-02-crypt\",\"total_bytes\":1099511627776,\"used_bytes\":290353347555}],\"observed_at_epoch\":1788520010,\"schema\":\"homelab-backup-observability-snapshot-v1\",\"targets\":[{\"archive_size_bytes\":478357303,\"id\":\"110\",\"kind\":\"VM\",\"logical_id\":\"vm110\"},{\"archive_size_bytes\":369892675,\"id\":\"111\",\"kind\":\"VM\",\"logical_id\":\"vm111\"},{\"archive_size_bytes\":223672749,\"id\":\"112\",\"kind\":\"VM\",\"logical_id\":\"vm112\"},{\"archive_size_bytes\":221529138,\"id\":\"113\",\"kind\":\"VM\",\"logical_id\":\"vm113\"},{\"archive_size_bytes\":10620176786,\"id\":\"130\",\"kind\":\"VM\",\"logical_id\":\"vm130\"},{\"archive_size_bytes\":887338816,\"id\":\"150\",\"kind\":\"VM\",\"logical_id\":\"vm150\"},{\"archive_size_bytes\":3571246511,\"id\":\"160\",\"kind\":\"VM\",\"logical_id\":\"vm160\"},{\"archive_size_bytes\":7974616162,\"id\":\"170\",\"kind\":\"VM\",\"logical_id\":\"vm170\"},{\"archive_size_bytes\":2028255985,\"id\":\"171\",\"kind\":\"VM\",\"logical_id\":\"vm171\"},{\"archive_size_bytes\":1557482946,\"id\":\"190\",\"kind\":\"VM\",\"logical_id\":\"vm190\"},{\"archive_size_bytes\":10998362942,\"id\":\"9130\",\"kind\":\"VM\",\"logical_id\":\"vm9130\"},{\"archive_size_bytes\":3197,\"id\":\"filebrowser\",\"kind\":\"APP\",\"logical_id\":\"app-filebrowser\"},{\"archive_size_bytes\":154716,\"id\":\"skladchik-reports-monitor\",\"kind\":\"APP\",\"logical_id\":\"app-skladchik-reports-monitor\"},{\"archive_size_bytes\":270193492,\"id\":\"immich\",\"kind\":\"APP\",\"logical_id\":\"app-immich\"},{\"archive_size_bytes\":75453412,\"id\":\"hbarhub\",\"kind\":\"APP\",\"logical_id\":\"app-hbarhub\"},{\"archive_size_bytes\":8970684,\"id\":\"control-bootstrap-v2\",\"kind\":\"CONTROL\",\"logical_id\":\"control-bootstrap-v2\"}]}\n{\"clouds\":[{\"free_bytes\":809158344080,\"remote\":\"pve01-mail-01-crypt\",\"total_bytes\":1099511627776,\"used_bytes\":290353283696},{\"free_bytes\":809158280221,\"remote\":\"pve01-mail-02-crypt\",\"total_bytes\":1099511627776,\"used_bytes\":290353347555}],\"observed_at_epoch\":1788525352,\"schema\":\"homelab-backup-observability-snapshot-v1\",\"targets\":[{\"archive_size_bytes\":478357303,\"id\":\"110\",\"kind\":\"VM\",\"logical_id\":\"vm110\"},{\"archive_size_bytes\":369892675,\"id\":\"111\",\"kind\":\"VM\",\"logical_id\":\"vm111\"},{\"archive_size_bytes\":223672749,\"id\":\"112\",\"kind\":\"VM\",\"logical_id\":\"vm112\"},{\"archive_size_bytes\":221529138,\"id\":\"113\",\"kind\":\"VM\",\"logical_id\":\"vm113\"},{\"archive_size_bytes\":10620176786,\"id\":\"130\",\"kind\":\"VM\",\"logical_id\":\"vm130\"},{\"archive_size_bytes\":887338816,\"id\":\"150\",\"kind\":\"VM\",\"logical_id\":\"vm150\"},{\"archive_size_bytes\":3571246511,\"id\":\"160\",\"kind\":\"VM\",\"logical_id\":\"vm160\"},{\"archive_size_bytes\":7974616162,\"id\":\"170\",\"kind\":\"VM\",\"logical_id\":\"vm170\"},{\"archive_size_bytes\":2028255985,\"id\":\"171\",\"kind\":\"VM\",\"logical_id\":\"vm171\"},{\"archive_size_bytes\":1557482946,\"id\":\"190\",\"kind\":\"VM\",\"logical_id\":\"vm190\"},{\"archive_size_bytes\":10998362942,\"id\":\"9130\",\"kind\":\"VM\",\"logical_id\":\"vm9130\"},{\"archive_size_bytes\":3197,\"id\":\"filebrowser\",\"kind\":\"APP\",\"logical_id\":\"app-filebrowser\"},{\"archive_size_bytes\":154716,\"id\":\"skladchik-reports-monitor\",\"kind\":\"APP\",\"logical_id\":\"app-skladchik-reports-monitor\"},{\"archive_size_bytes\":270193492,\"id\":\"immich\",\"kind\":\"APP\",\"logical_id\":\"app-immich\"},{\"archive_size_bytes\":75453412,\"id\":\"hbarhub\",\"kind\":\"APP\",\"logical_id\":\"app-hbarhub\"},{\"archive_size_bytes\":8970684,\"id\":\"control-bootstrap-v2\",\"kind\":\"CONTROL\",\"logical_id\":\"control-bootstrap-v2\"}]}\n{\"clouds\":[{\"free_bytes\":809158344080,\"remote\":\"pve01-mail-01-crypt\",\"total_bytes\":1099511627776,\"used_bytes\":290353283696},{\"free_bytes\":809158280221,\"remote\":\"pve01-mail-02-crypt\",\"total_bytes\":1099511627776,\"used_bytes\":290353347555}],\"observed_at_epoch\":1788528952,\"schema\":\"homelab-backup-observability-snapshot-v1\",\"targets\":[{\"archive_size_bytes\":478357303,\"id\":\"110\",\"kind\":\"VM\",\"logical_id\":\"vm110\"},{\"archive_size_bytes\":369892675,\"id\":\"111\",\"kind\":\"VM\",\"logical_id\":\"vm111\"},{\"archive_size_bytes\":223672749,\"id\":\"112\",\"kind\":\"VM\",\"logical_id\":\"vm112\"},{\"archive_size_bytes\":221529138,\"id\":\"113\",\"kind\":\"VM\",\"logical_id\":\"vm113\"},{\"archive_size_bytes\":10620176786,\"id\":\"130\",\"kind\":\"VM\",\"logical_id\":\"vm130\"},{\"archive_size_bytes\":887338816,\"id\":\"150\",\"kind\":\"VM\",\"logical_id\":\"vm150\"},{\"archive_size_bytes\":3571246511,\"id\":\"160\",\"kind\":\"VM\",\"logical_id\":\"vm160\"},{\"archive_size_bytes\":7974616162,\"id\":\"170\",\"kind\":\"VM\",\"logical_id\":\"vm170\"},{\"archive_size_bytes\":2028255985,\"id\":\"171\",\"kind\":\"VM\",\"logical_id\":\"vm171\"},{\"archive_size_bytes\":1557482946,\"id\":\"190\",\"kind\":\"VM\",\"logical_id\":\"vm190\"},{\"archive_size_bytes\":10998362942,\"id\":\"9130\",\"kind\":\"VM\",\"logical_id\":\"vm9130\"},{\"archive_size_bytes\":3197,\"id\":\"filebrowser\",\"kind\":\"APP\",\"logical_id\":\"app-filebrowser\"},{\"archive_size_bytes\":154716,\"id\":\"skladchik-reports-monitor\",\"kind\":\"APP\",\"logical_id\":\"app-skladchik-reports-monitor\"},{\"archive_size_bytes\":270193492,\"id\":\"immich\",\"kind\":\"APP\",\"logical_id\":\"app-immich\"},{\"archive_size_bytes\":75453412,\"id\":\"hbarhub\",\"kind\":\"APP\",\"logical_id\":\"app-hbarhub\"},{\"archive_size_bytes\":8970684,\"id\":\"control-bootstrap-v2\",\"kind\":\"CONTROL\",\"logical_id\":\"control-bootstrap-v2\"}]}\n{\"clouds\":[{\"free_bytes\":809158344080,\"remote\":\"pve01-mail-01-crypt\",\"total_bytes\":1099511627776,\"used_bytes\":290353283696},{\"free_bytes\":809158280221,\"remote\":\"pve01-mail-02-crypt\",\"total_bytes\":1099511627776,\"used_bytes\":290353347555}],\"observed_at_epoch\":1788532552,\"schema\":\"homelab-backup-observability-snapshot-v1\",\"targets\":[{\"archive_size_bytes\":478357303,\"id\":\"110\",\"kind\":\"VM\",\"logical_id\":\"vm110\"},{\"archive_size_bytes\":369892675,\"id\":\"111\",\"kind\":\"VM\",\"logical_id\":\"vm111\"},{\"archive_size_bytes\":223672749,\"id\":\"112\",\"kind\":\"VM\",\"logical_id\":\"vm112\"},{\"archive_size_bytes\":221529138,\"id\":\"113\",\"kind\":\"VM\",\"logical_id\":\"vm113\"},{\"archive_size_bytes\":10620176786,\"id\":\"130\",\"kind\":\"VM\",\"logical_id\":\"vm130\"},{\"archive_size_bytes\":887338816,\"id\":\"150\",\"kind\":\"VM\",\"logical_id\":\"vm150\"},{\"archive_size_bytes\":3571246511,\"id\":\"160\",\"kind\":\"VM\",\"logical_id\":\"vm160\"},{\"archive_size_bytes\":7974616162,\"id\":\"170\",\"kind\":\"VM\",\"logical_id\":\"vm170\"},{\"archive_size_bytes\":2028255985,\"id\":\"171\",\"kind\":\"VM\",\"logical_id\":\"vm171\"},{\"archive_size_bytes\":1557482946,\"id\":\"190\",\"kind\":\"VM\",\"logical_id\":\"vm190\"},{\"archive_size_bytes\":10998362942,\"id\":\"9130\",\"kind\":\"VM\",\"logical_id\":\"vm9130\"},{\"archive_size_bytes\":3197,\"id\":\"filebrowser\",\"kind\":\"APP\",\"logical_id\":\"app-filebrowser\"},{\"archive_size_bytes\":154716,\"id\":\"skladchik-reports-monitor\",\"kind\":\"APP\",\"logical_id\":\"app-skladchik-reports-monitor\"},{\"archive_size_bytes\":270193492,\"id\":\"immich\",\"kind\":\"APP\",\"logical_id\":\"app-immich\"},{\"archive_size_bytes\":75453412,\"id\":\"hbarhub\",\"kind\":\"APP\",\"logical_id\":\"app-hbarhub\"},{\"archive_size_bytes\":8970684,\"id\":\"control-bootstrap-v2\",\"kind\":\"CONTROL\",\"logical_id\":\"control-bootstrap-v2\"}]}\n{\"clouds\":[{\"free_bytes\":809158344080,\"remote\":\"pve01-mail-01-crypt\",\"total_bytes\":1099511627776,\"used_bytes\":290353283696},{\"free_bytes\":809158280221,\"remote\":\"pve01-mail-02-crypt\",\"total_bytes\":1099511627776,\"used_bytes\":290353347555}],\"observed_at_epoch\":1788536192,\"schema\":\"homelab-backup-observability-snapshot-v1\",\"targets\":[{\"archive_size_bytes\":478357303,\"id\":\"110\",\"kind\":\"VM\",\"logical_id\":\"vm110\"},{\"archive_size_bytes\":369892675,\"id\":\"111\",\"kind\":\"VM\",\"logical_id\":\"vm111\"},{\"archive_size_bytes\":223672749,\"id\":\"112\",\"kind\":\"VM\",\"logical_id\":\"vm112\"},{\"archive_size_bytes\":221529138,\"id\":\"113\",\"kind\":\"VM\",\"logical_id\":\"vm113\"},{\"archive_size_bytes\":10620176786,\"id\":\"130\",\"kind\":\"VM\",\"logical_id\":\"vm130\"},{\"archive_size_bytes\":887338816,\"id\":\"150\",\"kind\":\"VM\",\"logical_id\":\"vm150\"},{\"archive_size_bytes\":3571246511,\"id\":\"160\",\"kind\":\"VM\",\"logical_id\":\"vm160\"},{\"archive_size_bytes\":7974616162,\"id\":\"170\",\"kind\":\"VM\",\"logical_id\":\"vm170\"},{\"archive_size_bytes\":2028255985,\"id\":\"171\",\"kind\":\"VM\",\"logical_id\":\"vm171\"},{\"archive_size_bytes\":1557482946,\"id\":\"190\",\"kind\":\"VM\",\"logical_id\":\"vm190\"},{\"archive_size_bytes\":10998362942,\"id\":\"9130\",\"kind\":\"VM\",\"logical_id\":\"vm9130\"},{\"archive_size_bytes\":3197,\"id\":\"filebrowser\",\"kind\":\"APP\",\"logical_id\":\"app-filebrowser\"},{\"archive_size_bytes\":154716,\"id\":\"skladchik-reports-monitor\",\"kind\":\"APP\",\"logical_id\":\"app-skladchik-reports-monitor\"},{\"archive_size_bytes\":270193492,\"id\":\"immich\",\"kind\":\"APP\",\"logical_id\":\"app-immich\"},{\"archive_size_bytes\":75453412,\"id\":\"hbarhub\",\"kind\":\"APP\",\"logical_id\":\"app-hbarhub\"},{\"archive_size_bytes\":8970684,\"id\":\"control-bootstrap-v2\",\"kind\":\"CONTROL\",\"logical_id\":\"control-bootstrap-v2\"}]}\n{\"clouds\":[{\"free_bytes\":809158344080,\"remote\":\"pve01-mail-01-crypt\",\"total_bytes\":1099511627776,\"used_bytes\":290353283696},{\"free_bytes\":809158280221,\"remote\":\"pve01-mail-02-crypt\",\"total_bytes\":1099511627776,\"used_bytes\":290353347555}],\"observed_at_epoch\":1788541532,\"schema\":\"homelab-backup-observability-snapshot-v1\",\"targets\":[{\"archive_size_bytes\":478357303,\"id\":\"110\",\"kind\":\"VM\",\"logical_id\":\"vm110\"},{\"archive_size_bytes\":369892675,\"id\":\"111\",\"kind\":\"VM\",\"logical_id\":\"vm111\"},{\"archive_size_bytes\":223672749,\"id\":\"112\",\"kind\":\"VM\",\"logical_id\":\"vm112\"},{\"archive_size_bytes\":221529138,\"id\":\"113\",\"kind\":\"VM\",\"logical_id\":\"vm113\"},{\"archive_size_bytes\":10620176786,\"id\":\"130\",\"kind\":\"VM\",\"logical_id\":\"vm130\"},{\"archive_size_bytes\":887338816,\"id\":\"150\",\"kind\":\"VM\",\"logical_id\":\"vm150\"},{\"archive_size_bytes\":3571246511,\"id\":\"160\",\"kind\":\"VM\",\"logical_id\":\"vm160\"},{\"archive_size_bytes\":7974616162,\"id\":\"170\",\"kind\":\"VM\",\"logical_id\":\"vm170\"},{\"archive_size_bytes\":2028255985,\"id\":\"171\",\"kind\":\"VM\",\"logical_id\":\"vm171\"},{\"archive_size_bytes\":1557482946,\"id\":\"190\",\"kind\":\"VM\",\"logical_id\":\"vm190\"},{\"archive_size_bytes\":10998362942,\"id\":\"9130\",\"kind\":\"VM\",\"logical_id\":\"vm9130\"},{\"archive_size_bytes\":3197,\"id\":\"filebrowser\",\"kind\":\"APP\",\"logical_id\":\"app-filebrowser\"},{\"archive_size_bytes\":154716,\"id\":\"skladchik-reports-monitor\",\"kind\":\"APP\",\"logical_id\":\"app-skladchik-reports-monitor\"},{\"archive_size_bytes\":270193492,\"id\":\"immich\",\"kind\":\"APP\",\"logical_id\":\"app-immich\"},{\"archive_size_bytes\":75453412,\"id\":\"hbarhub\",\"kind\":\"APP\",\"logical_id\":\"app-hbarhub\"},{\"archive_size_bytes\":8970684,\"id\":\"control-bootstrap-v2\",\"kind\":\"CONTROL\",\"logical_id\":\"control-bootstrap-v2\"}]}\n{\"clouds\":[{\"free_bytes\":809158344080,\"remote\":\"pve01-mail-01-crypt\",\"total_bytes\":1099511627776,\"used_bytes\":290353283696},{\"free_bytes\":809158280221,\"remote\":\"pve01-mail-02-crypt\",\"total_bytes\":1099511627776,\"used_bytes\":290353347555}],\"observed_at_epoch\":1788545210,\"schema\":\"homelab-backup-observability-snapshot-v1\",\"targets\":[{\"archive_size_bytes\":478357303,\"id\":\"110\",\"kind\":\"VM\",\"logical_id\":\"vm110\"},{\"archive_size_bytes\":369892675,\"id\":\"111\",\"kind\":\"VM\",\"logical_id\":\"vm111\"},{\"archive_size_bytes\":223672749,\"id\":\"112\",\"kind\":\"VM\",\"logical_id\":\"vm112\"},{\"archive_size_bytes\":221529138,\"id\":\"113\",\"kind\":\"VM\",\"logical_id\":\"vm113\"},{\"archive_size_bytes\":10620176786,\"id\":\"130\",\"kind\":\"VM\",\"logical_id\":\"vm130\"},{\"archive_size_bytes\":887338816,\"id\":\"150\",\"kind\":\"VM\",\"logical_id\":\"vm150\"},{\"archive_size_bytes\":3571246511,\"id\":\"160\",\"kind\":\"VM\",\"logical_id\":\"vm160\"},{\"archive_size_bytes\":7974616162,\"id\":\"170\",\"kind\":\"VM\",\"logical_id\":\"vm170\"},{\"archive_size_bytes\":2028255985,\"id\":\"171\",\"kind\":\"VM\",\"logical_id\":\"vm171\"},{\"archive_size_bytes\":1557482946,\"id\":\"190\",\"kind\":\"VM\",\"logical_id\":\"vm190\"},{\"archive_size_bytes\":10998362942,\"id\":\"9130\",\"kind\":\"VM\",\"logical_id\":\"vm9130\"},{\"archive_size_bytes\":3197,\"id\":\"filebrowser\",\"kind\":\"APP\",\"logical_id\":\"app-filebrowser\"},{\"archive_size_bytes\":154716,\"id\":\"skladchik-reports-monitor\",\"kind\":\"APP\",\"logical_id\":\"app-skladchik-reports-monitor\"},{\"archive_size_bytes\":270193492,\"id\":\"immich\",\"kind\":\"APP\",\"logical_id\":\"app-immich\"},{\"archive_size_bytes\":75453412,\"id\":\"hbarhub\",\"kind\":\"APP\",\"logical_id\":\"app-hbarhub\"},{\"archive_size_bytes\":8970684,\"id\":\"control-bootstrap-v2\",\"kind\":\"CONTROL\",\"logical_id\":\"control-bootstrap-v2\"}]}\n{\"clouds\":[{\"free_bytes\":789729590488,\"remote\":\"pve01-mail-01-crypt\",\"total_bytes\":1099511627776,\"used_bytes\":309782037288},{\"free_bytes\":789729526629,\"remote\":\"pve01-mail-02-crypt\",\"total_bytes\":1099511627776,\"used_bytes\":309782101147}],\"observed_at_epoch\":1788550592,\"schema\":\"homelab-backup-observability-snapshot-v1\",\"targets\":[{\"archive_size_bytes\":478357303,\"id\":\"110\",\"kind\":\"VM\",\"logical_id\":\"vm110\"},{\"archive_size_bytes\":369892675,\"id\":\"111\",\"kind\":\"VM\",\"logical_id\":\"vm111\"},{\"archive_size_bytes\":223672749,\"id\":\"112\",\"kind\":\"VM\",\"logical_id\":\"vm112\"},{\"archive_size_bytes\":221529138,\"id\":\"113\",\"kind\":\"VM\",\"logical_id\":\"vm113\"},{\"archive_size_bytes\":10614525791,\"id\":\"130\",\"kind\":\"VM\",\"logical_id\":\"vm130\"},{\"archive_size_bytes\":887338816,\"id\":\"150\",\"kind\":\"VM\",\"logical_id\":\"vm150\"},{\"archive_size_bytes\":3571246511,\"id\":\"160\",\"kind\":\"VM\",\"logical_id\":\"vm160\"},{\"archive_size_bytes\":6440615330,\"id\":\"170\",\"kind\":\"VM\",\"logical_id\":\"vm170\"},{\"archive_size_bytes\":1557482946,\"id\":\"190\",\"kind\":\"VM\",\"logical_id\":\"vm190\"},{\"archive_size_bytes\":10998362942,\"id\":\"9130\",\"kind\":\"VM\",\"logical_id\":\"vm9130\"},{\"archive_size_bytes\":3182,\"id\":\"filebrowser\",\"kind\":\"APP\",\"logical_id\":\"app-filebrowser\"},{\"archive_size_bytes\":156203,\"id\":\"skladchik-reports-monitor\",\"kind\":\"APP\",\"logical_id\":\"app-skladchik-reports-monitor\"},{\"archive_size_bytes\":270193546,\"id\":\"immich\",\"kind\":\"APP\",\"logical_id\":\"app-immich\"},{\"archive_size_bytes\":75522547,\"id\":\"hbarhub\",\"kind\":\"APP\",\"logical_id\":\"app-hbarhub\"},{\"archive_size_bytes\":8970684,\"id\":\"control-bootstrap-v2\",\"kind\":\"CONTROL\",\"logical_id\":\"control-bootstrap-v2\"}]}\n{\"clouds\":[{\"free_bytes\":789729589363,\"remote\":\"pve01-mail-01-crypt\",\"total_bytes\":1099511627776,\"used_bytes\":309782038413},{\"free_bytes\":789729525504,\"remote\":\"pve01-mail-02-crypt\",\"total_bytes\":1099511627776,\"used_bytes\":309782102272}],\"observed_at_epoch\":1788554210,\"schema\":\"homelab-backup-observability-snapshot-v1\",\"targets\":[{\"archive_size_bytes\":478357303,\"id\":\"110\",\"kind\":\"VM\",\"logical_id\":\"vm110\"},{\"archive_size_bytes\":369892675,\"id\":\"111\",\"kind\":\"VM\",\"logical_id\":\"vm111\"},{\"archive_size_bytes\":223672749,\"id\":\"112\",\"kind\":\"VM\",\"logical_id\":\"vm112\"},{\"archive_size_bytes\":221529138,\"id\":\"113\",\"kind\":\"VM\",\"logical_id\":\"vm113\"},{\"archive_size_bytes\":10614525791,\"id\":\"130\",\"kind\":\"VM\",\"logical_id\":\"vm130\"},{\"archive_size_bytes\":887338816,\"id\":\"150\",\"kind\":\"VM\",\"logical_id\":\"vm150\"},{\"archive_size_bytes\":3571246511,\"id\":\"160\",\"kind\":\"VM\",\"logical_id\":\"vm160\"},{\"archive_size_bytes\":6440615330,\"id\":\"170\",\"kind\":\"VM\",\"logical_id\":\"vm170\"},{\"archive_size_bytes\":2022982219,\"id\":\"171\",\"kind\":\"VM\",\"logical_id\":\"vm171\"},{\"archive_size_bytes\":1557482946,\"id\":\"190\",\"kind\":\"VM\",\"logical_id\":\"vm190\"},{\"archive_size_bytes\":10998362942,\"id\":\"9130\",\"kind\":\"VM\",\"logical_id\":\"vm9130\"},{\"archive_size_bytes\":3182,\"id\":\"filebrowser\",\"kind\":\"APP\",\"logical_id\":\"app-filebrowser\"},{\"archive_size_bytes\":156203,\"id\":\"skladchik-reports-monitor\",\"kind\":\"APP\",\"logical_id\":\"app-skladchik-reports-monitor\"},{\"archive_size_bytes\":270193546,\"id\":\"immich\",\"kind\":\"APP\",\"logical_id\":\"app-immich\"},{\"archive_size_bytes\":75522547,\"id\":\"hbarhub\",\"kind\":\"APP\",\"logical_id\":\"app-hbarhub\"},{\"archive_size_bytes\":8970684,\"id\":\"control-bootstrap-v2\",\"kind\":\"CONTROL\",\"logical_id\":\"control-bootstrap-v2\"}]}\n{\"clouds\":[{\"free_bytes\":786104095977,\"remote\":\"pve01-mail-01-crypt\",\"total_bytes\":1099511627776,\"used_bytes\":313407531799},{\"free_bytes\":786113009899,\"remote\":\"pve01-mail-02-crypt\",\"total_bytes\":1099511627776,\"used_bytes\":313398617877}],\"observed_at_epoch\":1788559610,\"schema\":\"homelab-backup-observability-snapshot-v1\",\"targets\":[{\"archive_size_bytes\":478357303,\"id\":\"110\",\"kind\":\"VM\",\"logical_id\":\"vm110\"},{\"archive_size_bytes\":369892675,\"id\":\"111\",\"kind\":\"VM\",\"logical_id\":\"vm111\"},{\"archive_size_bytes\":223672749,\"id\":\"112\",\"kind\":\"VM\",\"logical_id\":\"vm112\"},{\"archive_size_bytes\":221529138,\"id\":\"113\",\"kind\":\"VM\",\"logical_id\":\"vm113\"},{\"archive_size_bytes\":10614525791,\"id\":\"130\",\"kind\":\"VM\",\"logical_id\":\"vm130\"},{\"archive_size_bytes\":887338816,\"id\":\"150\",\"kind\":\"VM\",\"logical_id\":\"vm150\"},{\"archive_size_bytes\":3615630758,\"id\":\"160\",\"kind\":\"VM\",\"logical_id\":\"vm160\"},{\"archive_size_bytes\":6440615330,\"id\":\"170\",\"kind\":\"VM\",\"logical_id\":\"vm170\"},{\"archive_size_bytes\":2022982219,\"id\":\"171\",\"kind\":\"VM\",\"logical_id\":\"vm171\"},{\"archive_size_bytes\":1557482946,\"id\":\"190\",\"kind\":\"VM\",\"logical_id\":\"vm190\"},{\"archive_size_bytes\":10998362942,\"id\":\"9130\",\"kind\":\"VM\",\"logical_id\":\"vm9130\"},{\"archive_size_bytes\":3182,\"id\":\"filebrowser\",\"kind\":\"APP\",\"logical_id\":\"app-filebrowser\"},{\"archive_size_bytes\":156203,\"id\":\"skladchik-reports-monitor\",\"kind\":\"APP\",\"logical_id\":\"app-skladchik-reports-monitor\"},{\"archive_size_bytes\":270193546,\"id\":\"immich\",\"kind\":\"APP\",\"logical_id\":\"app-immich\"},{\"archive_size_bytes\":75522547,\"id\":\"hbarhub\",\"kind\":\"APP\",\"logical_id\":\"app-hbarhub\"}]}\n{\"clouds\":[{\"free_bytes\":786104094818,\"remote\":\"pve01-mail-01-crypt\",\"total_bytes\":1099511627776,\"used_bytes\":313407532958},{\"free_bytes\":786104030959,\"remote\":\"pve01-mail-02-crypt\",\"total_bytes\":1099511627776,\"used_bytes\":313407596817}],\"observed_at_epoch\":1788564916,\"schema\":\"homelab-backup-observability-snapshot-v1\",\"targets\":[{\"archive_size_bytes\":478357303,\"id\":\"110\",\"kind\":\"VM\",\"logical_id\":\"vm110\"},{\"archive_size_bytes\":369892675,\"id\":\"111\",\"kind\":\"VM\",\"logical_id\":\"vm111\"},{\"archive_size_bytes\":223672749,\"id\":\"112\",\"kind\":\"VM\",\"logical_id\":\"vm112\"},{\"archive_size_bytes\":221529138,\"id\":\"113\",\"kind\":\"VM\",\"logical_id\":\"vm113\"},{\"archive_size_bytes\":10614525791,\"id\":\"130\",\"kind\":\"VM\",\"logical_id\":\"vm130\"},{\"archive_size_bytes\":887338816,\"id\":\"150\",\"kind\":\"VM\",\"logical_id\":\"vm150\"},{\"archive_size_bytes\":3615630758,\"id\":\"160\",\"kind\":\"VM\",\"logical_id\":\"vm160\"},{\"archive_size_bytes\":6440615330,\"id\":\"170\",\"kind\":\"VM\",\"logical_id\":\"vm170\"},{\"archive_size_bytes\":2022982219,\"id\":\"171\",\"kind\":\"VM\",\"logical_id\":\"vm171\"},{\"archive_size_bytes\":1557482946,\"id\":\"190\",\"kind\":\"VM\",\"logical_id\":\"vm190\"},{\"archive_size_bytes\":10998362942,\"id\":\"9130\",\"kind\":\"VM\",\"logical_id\":\"vm9130\"},{\"archive_size_bytes\":3182,\"id\":\"filebrowser\",\"kind\":\"APP\",\"logical_id\":\"app-filebrowser\"},{\"archive_size_bytes\":156203,\"id\":\"skladchik-reports-monitor\",\"kind\":\"APP\",\"logical_id\":\"app-skladchik-reports-monitor\"},{\"archive_size_bytes\":270193546,\"id\":\"immich\",\"kind\":\"APP\",\"logical_id\":\"app-immich\"},{\"archive_size_bytes\":75522547,\"id\":\"hbarhub\",\"kind\":\"APP\",\"logical_id\":\"app-hbarhub\"},{\"archive_size_bytes\":8975409,\"id\":\"control-bootstrap-v2\",\"kind\":\"CONTROL\",\"logical_id\":\"control-bootstrap-v2\"}]}\n{\"clouds\":[{\"free_bytes\":786104094818,\"remote\":\"pve01-mail-01-crypt\",\"total_bytes\":1099511627776,\"used_bytes\":313407532958},{\"free_bytes\":786104030959,\"remote\":\"pve01-mail-02-crypt\",\"total_bytes\":1099511627776,\"used_bytes\":313407596817}],\"observed_at_epoch\":1788568552,\"schema\":\"homelab-backup-observability-snapshot-v1\",\"targets\":[{\"archive_size_bytes\":478357303,\"id\":\"110\",\"kind\":\"VM\",\"logical_id\":\"vm110\"},{\"archive_size_bytes\":369892675,\"id\":\"111\",\"kind\":\"VM\",\"logical_id\":\"vm111\"},{\"archive_size_bytes\":223672749,\"id\":\"112\",\"kind\":\"VM\",\"logical_id\":\"vm112\"},{\"archive_size_bytes\":221529138,\"id\":\"113\",\"kind\":\"VM\",\"logical_id\":\"vm113\"},{\"archive_size_bytes\":10614525791,\"id\":\"130\",\"kind\":\"VM\",\"logical_id\":\"vm130\"},{\"archive_size_bytes\":887338816,\"id\":\"150\",\"kind\":\"VM\",\"logical_id\":\"vm150\"},{\"archive_size_bytes\":3615630758,\"id\":\"160\",\"kind\":\"VM\",\"logical_id\":\"vm160\"},{\"archive_size_bytes\":6440615330,\"id\":\"170\",\"kind\":\"VM\",\"logical_id\":\"vm170\"},{\"archive_size_bytes\":2022982219,\"id\":\"171\",\"kind\":\"VM\",\"logical_id\":\"vm171\"},{\"archive_size_bytes\":1557482946,\"id\":\"190\",\"kind\":\"VM\",\"logical_id\":\"vm190\"},{\"archive_size_bytes\":10998362942,\"id\":\"9130\",\"kind\":\"VM\",\"logical_id\":\"vm9130\"},{\"archive_size_bytes\":3182,\"id\":\"filebrowser\",\"kind\":\"APP\",\"logical_id\":\"app-filebrowser\"},{\"archive_size_bytes\":156203,\"id\":\"skladchik-reports-monitor\",\"kind\":\"APP\",\"logical_id\":\"app-skladchik-reports-monitor\"},{\"archive_size_bytes\":270193546,\"id\":\"immich\",\"kind\":\"APP\",\"logical_id\":\"app-immich\"},{\"archive_size_bytes\":75522547,\"id\":\"hbarhub\",\"kind\":\"APP\",\"logical_id\":\"app-hbarhub\"},{\"archive_size_bytes\":8975409,\"id\":\"control-bootstrap-v2\",\"kind\":\"CONTROL\",\"logical_id\":\"control-bootstrap-v2\"}]}\n{\"clouds\":[{\"free_bytes\":786104094818,\"remote\":\"pve01-mail-01-crypt\",\"total_bytes\":1099511627776,\"used_bytes\":313407532958},{\"free_bytes\":786104030959,\"remote\":\"pve01-mail-02-crypt\",\"total_bytes\":1099511627776,\"used_bytes\":313407596817}],\"observed_at_epoch\":1788572153,\"schema\":\"homelab-backup-observability-snapshot-v1\",\"targets\":[{\"archive_size_bytes\":478357303,\"id\":\"110\",\"kind\":\"VM\",\"logical_id\":\"vm110\"},{\"archive_size_bytes\":369892675,\"id\":\"111\",\"kind\":\"VM\",\"logical_id\":\"vm111\"},{\"archive_size_bytes\":223672749,\"id\":\"112\",\"kind\":\"VM\",\"logical_id\":\"vm112\"},{\"archive_size_bytes\":221529138,\"id\":\"113\",\"kind\":\"VM\",\"logical_id\":\"vm113\"},{\"archive_size_bytes\":10614525791,\"id\":\"130\",\"kind\":\"VM\",\"logical_id\":\"vm130\"},{\"archive_size_bytes\":887338816,\"id\":\"150\",\"kind\":\"VM\",\"logical_id\":\"vm150\"},{\"archive_size_bytes\":3615630758,\"id\":\"160\",\"kind\":\"VM\",\"logical_id\":\"vm160\"},{\"archive_size_bytes\":6440615330,\"id\":\"170\",\"kind\":\"VM\",\"logical_id\":\"vm170\"},{\"archive_size_bytes\":2022982219,\"id\":\"171\",\"kind\":\"VM\",\"logical_id\":\"vm171\"},{\"archive_size_bytes\":1557482946,\"id\":\"190\",\"kind\":\"VM\",\"logical_id\":\"vm190\"},{\"archive_size_bytes\":10998362942,\"id\":\"9130\",\"kind\":\"VM\",\"logical_id\":\"vm9130\"},{\"archive_size_bytes\":3182,\"id\":\"filebrowser\",\"kind\":\"APP\",\"logical_id\":\"app-filebrowser\"},{\"archive_size_bytes\":156203,\"id\":\"skladchik-reports-monitor\",\"kind\":\"APP\",\"logical_id\":\"app-skladchik-reports-monitor\"},{\"archive_size_bytes\":270193546,\"id\":\"immich\",\"kind\":\"APP\",\"logical_id\":\"app-immich\"},{\"archive_size_bytes\":75522547,\"id\":\"hbarhub\",\"kind\":\"APP\",\"logical_id\":\"app-hbarhub\"},{\"archive_size_bytes\":8975409,\"id\":\"control-bootstrap-v2\",\"kind\":\"CONTROL\",\"logical_id\":\"control-bootstrap-v2\"}]}\n{\"clouds\":[{\"free_bytes\":786104094818,\"remote\":\"pve01-mail-01-crypt\",\"total_bytes\":1099511627776,\"used_bytes\":313407532958},{\"free_bytes\":786104030959,\"remote\":\"pve01-mail-02-crypt\",\"total_bytes\":1099511627776,\"used_bytes\":313407596817}],\"observed_at_epoch\":1788575770,\"schema\":\"homelab-backup-observability-snapshot-v1\",\"targets\":[{\"archive_size_bytes\":478357303,\"id\":\"110\",\"kind\":\"VM\",\"logical_id\":\"vm110\"},{\"archive_size_bytes\":369892675,\"id\":\"111\",\"kind\":\"VM\",\"logical_id\":\"vm111\"},{\"archive_size_bytes\":223672749,\"id\":\"112\",\"kind\":\"VM\",\"logical_id\":\"vm112\"},{\"archive_size_bytes\":221529138,\"id\":\"113\",\"kind\":\"VM\",\"logical_id\":\"vm113\"},{\"archive_size_bytes\":10614525791,\"id\":\"130\",\"kind\":\"VM\",\"logical_id\":\"vm130\"},{\"archive_size_bytes\":887338816,\"id\":\"150\",\"kind\":\"VM\",\"logical_id\":\"vm150\"},{\"archive_size_bytes\":3615630758,\"id\":\"160\",\"kind\":\"VM\",\"logical_id\":\"vm160\"},{\"archive_size_bytes\":6440615330,\"id\":\"170\",\"kind\":\"VM\",\"logical_id\":\"vm170\"},{\"archive_size_bytes\":2022982219,\"id\":\"171\",\"kind\":\"VM\",\"logical_id\":\"vm171\"},{\"archive_size_bytes\":1557482946,\"id\":\"190\",\"kind\":\"VM\",\"logical_id\":\"vm190\"},{\"archive_size_bytes\":10998362942,\"id\":\"9130\",\"kind\":\"VM\",\"logical_id\":\"vm9130\"},{\"archive_size_bytes\":3182,\"id\":\"filebrowser\",\"kind\":\"APP\",\"logical_id\":\"app-filebrowser\"},{\"archive_size_bytes\":156203,\"id\":\"skladchik-reports-monitor\",\"kind\":\"APP\",\"logical_id\":\"app-skladchik-reports-monitor\"},{\"archive_size_bytes\":270193546,\"id\":\"immich\",\"kind\":\"APP\",\"logical_id\":\"app-immich\"},{\"archive_size_bytes\":75522547,\"id\":\"hbarhub\",\"kind\":\"APP\",\"logical_id\":\"app-hbarhub\"},{\"archive_size_bytes\":8975409,\"id\":\"control-bootstrap-v2\",\"kind\":\"CONTROL\",\"logical_id\":\"control-bootstrap-v2\"}]}\n{\"clouds\":[{\"free_bytes\":786104094818,\"remote\":\"pve01-mail-01-crypt\",\"total_bytes\":1099511627776,\"used_bytes\":313407532958},{\"free_bytes\":786104030959,\"remote\":\"pve01-mail-02-crypt\",\"total_bytes\":1099511627776,\"used_bytes\":313407596817}],\"observed_at_epoch\":1788579413,\"schema\":\"homelab-backup-observability-snapshot-v1\",\"targets\":[{\"archive_size_bytes\":478357303,\"id\":\"110\",\"kind\":\"VM\",\"logical_id\":\"vm110\"},{\"archive_size_bytes\":369892675,\"id\":\"111\",\"kind\":\"VM\",\"logical_id\":\"vm111\"},{\"archive_size_bytes\":223672749,\"id\":\"112\",\"kind\":\"VM\",\"logical_id\":\"vm112\"},{\"archive_size_bytes\":221529138,\"id\":\"113\",\"kind\":\"VM\",\"logical_id\":\"vm113\"},{\"archive_size_bytes\":10614525791,\"id\":\"130\",\"kind\":\"VM\",\"logical_id\":\"vm130\"},{\"archive_size_bytes\":887338816,\"id\":\"150\",\"kind\":\"VM\",\"logical_id\":\"vm150\"},{\"archive_size_bytes\":3615630758,\"id\":\"160\",\"kind\":\"VM\",\"logical_id\":\"vm160\"},{\"archive_size_bytes\":6440615330,\"id\":\"170\",\"kind\":\"VM\",\"logical_id\":\"vm170\"},{\"archive_size_bytes\":2022982219,\"id\":\"171\",\"kind\":\"VM\",\"logical_id\":\"vm171\"},{\"archive_size_bytes\":1557482946,\"id\":\"190\",\"kind\":\"VM\",\"logical_id\":\"vm190\"},{\"archive_size_bytes\":10998362942,\"id\":\"9130\",\"kind\":\"VM\",\"logical_id\":\"vm9130\"},{\"archive_size_bytes\":3182,\"id\":\"filebrowser\",\"kind\":\"APP\",\"logical_id\":\"app-filebrowser\"},{\"archive_size_bytes\":156203,\"id\":\"skladchik-reports-monitor\",\"kind\":\"APP\",\"logical_id\":\"app-skladchik-reports-monitor\"},{\"archive_size_bytes\":270193546,\"id\":\"immich\",\"kind\":\"APP\",\"logical_id\":\"app-immich\"},{\"archive_size_bytes\":75522547,\"id\":\"hbarhub\",\"kind\":\"APP\",\"logical_id\":\"app-hbarhub\"},{\"archive_size_bytes\":8975409,\"id\":\"control-bootstrap-v2\",\"kind\":\"CONTROL\",\"logical_id\":\"control-bootstrap-v2\"}]}\n{\"clouds\":[{\"free_bytes\":786104094818,\"remote\":\"pve01-mail-01-crypt\",\"total_bytes\":1099511627776,\"used_bytes\":313407532958},{\"free_bytes\":786104030959,\"remote\":\"pve01-mail-02-crypt\",\"total_bytes\":1099511627776,\"used_bytes\":313407596817}],\"observed_at_epoch\":1788583036,\"schema\":\"homelab-backup-observability-snapshot-v1\",\"targets\":[{\"archive_size_bytes\":478357303,\"id\":\"110\",\"kind\":\"VM\",\"logical_id\":\"vm110\"},{\"archive_size_bytes\":369892675,\"id\":\"111\",\"kind\":\"VM\",\"logical_id\":\"vm111\"},{\"archive_size_bytes\":223672749,\"id\":\"112\",\"kind\":\"VM\",\"logical_id\":\"vm112\"},{\"archive_size_bytes\":221529138,\"id\":\"113\",\"kind\":\"VM\",\"logical_id\":\"vm113\"},{\"archive_size_bytes\":10614525791,\"id\":\"130\",\"kind\":\"VM\",\"logical_id\":\"vm130\"},{\"archive_size_bytes\":887338816,\"id\":\"150\",\"kind\":\"VM\",\"logical_id\":\"vm150\"},{\"archive_size_bytes\":3615630758,\"id\":\"160\",\"kind\":\"VM\",\"logical_id\":\"vm160\"},{\"archive_size_bytes\":6440615330,\"id\":\"170\",\"kind\":\"VM\",\"logical_id\":\"vm170\"},{\"archive_size_bytes\":2022982219,\"id\":\"171\",\"kind\":\"VM\",\"logical_id\":\"vm171\"},{\"archive_size_bytes\":1557482946,\"id\":\"190\",\"kind\":\"VM\",\"logical_id\":\"vm190\"},{\"archive_size_bytes\":10998362942,\"id\":\"9130\",\"kind\":\"VM\",\"logical_id\":\"vm9130\"},{\"archive_size_bytes\":3182,\"id\":\"filebrowser\",\"kind\":\"APP\",\"logical_id\":\"app-filebrowser\"},{\"archive_size_bytes\":156203,\"id\":\"skladchik-reports-monitor\",\"kind\":\"APP\",\"logical_id\":\"app-skladchik-reports-monitor\"},{\"archive_size_bytes\":270193546,\"id\":\"immich\",\"kind\":\"APP\",\"logical_id\":\"app-immich\"},{\"archive_size_bytes\":75522547,\"id\":\"hbarhub\",\"kind\":\"APP\",\"logical_id\":\"app-hbarhub\"},{\"archive_size_bytes\":8975409,\"id\":\"control-bootstrap-v2\",\"kind\":\"CONTROL\",\"logical_id\":\"control-bootstrap-v2\"}]}\n{\"clouds\":[{\"free_bytes\":786104095252,\"remote\":\"pve01-mail-01-crypt\",\"total_bytes\":1099511627776,\"used_bytes\":313407532524},{\"free_bytes\":786104031393,\"remote\":\"pve01-mail-02-crypt\",\"total_bytes\":1099511627776,\"used_bytes\":313407596383}],\"observed_at_epoch\":1788588332,\"schema\":\"homelab-backup-observability-snapshot-v1\",\"targets\":[{\"archive_size_bytes\":478357303,\"id\":\"110\",\"kind\":\"VM\",\"logical_id\":\"vm110\"},{\"archive_size_bytes\":369892675,\"id\":\"111\",\"kind\":\"VM\",\"logical_id\":\"vm111\"},{\"archive_size_bytes\":223672749,\"id\":\"112\",\"kind\":\"VM\",\"logical_id\":\"vm112\"},{\"archive_size_bytes\":221529138,\"id\":\"113\",\"kind\":\"VM\",\"logical_id\":\"vm113\"},{\"archive_size_bytes\":10614525791,\"id\":\"130\",\"kind\":\"VM\",\"logical_id\":\"vm130\"},{\"archive_size_bytes\":887338816,\"id\":\"150\",\"kind\":\"VM\",\"logical_id\":\"vm150\"},{\"archive_size_bytes\":3615630758,\"id\":\"160\",\"kind\":\"VM\",\"logical_id\":\"vm160\"},{\"archive_size_bytes\":6440615330,\"id\":\"170\",\"kind\":\"VM\",\"logical_id\":\"vm170\"},{\"archive_size_bytes\":2022982219,\"id\":\"171\",\"kind\":\"VM\",\"logical_id\":\"vm171\"},{\"archive_size_bytes\":1557482946,\"id\":\"190\",\"kind\":\"VM\",\"logical_id\":\"vm190\"},{\"archive_size_bytes\":10998362942,\"id\":\"9130\",\"kind\":\"VM\",\"logical_id\":\"vm9130\"},{\"archive_size_bytes\":3182,\"id\":\"filebrowser\",\"kind\":\"APP\",\"logical_id\":\"app-filebrowser\"},{\"archive_size_bytes\":156203,\"id\":\"skladchik-reports-monitor\",\"kind\":\"APP\",\"logical_id\":\"app-skladchik-reports-monitor\"},{\"archive_size_bytes\":270193546,\"id\":\"immich\",\"kind\":\"APP\",\"logical_id\":\"app-immich\"},{\"archive_size_bytes\":75522547,\"id\":\"hbarhub\",\"kind\":\"APP\",\"logical_id\":\"app-hbarhub\"},{\"archive_size_bytes\":8975409,\"id\":\"control-bootstrap-v2\",\"kind\":\"CONTROL\",\"logical_id\":\"control-bootstrap-v2\"}]}\n{\"clouds\":[{\"free_bytes\":786104095252,\"remote\":\"pve01-mail-01-crypt\",\"total_bytes\":1099511627776,\"used_bytes\":313407532524},{\"free_bytes\":786104031393,\"remote\":\"pve01-mail-02-crypt\",\"total_bytes\":1099511627776,\"used_bytes\":313407596383}],\"observed_at_epoch\":1788591953,\"schema\":\"homelab-backup-observability-snapshot-v1\",\"targets\":[{\"archive_size_bytes\":478357303,\"id\":\"110\",\"kind\":\"VM\",\"logical_id\":\"vm110\"},{\"archive_size_bytes\":369892675,\"id\":\"111\",\"kind\":\"VM\",\"logical_id\":\"vm111\"},{\"archive_size_bytes\":223672749,\"id\":\"112\",\"kind\":\"VM\",\"logical_id\":\"vm112\"},{\"archive_size_bytes\":221529138,\"id\":\"113\",\"kind\":\"VM\",\"logical_id\":\"vm113\"},{\"archive_size_bytes\":10614525791,\"id\":\"130\",\"kind\":\"VM\",\"logical_id\":\"vm130\"},{\"archive_size_bytes\":887338816,\"id\":\"150\",\"kind\":\"VM\",\"logical_id\":\"vm150\"},{\"archive_size_bytes\":3615630758,\"id\":\"160\",\"kind\":\"VM\",\"logical_id\":\"vm160\"},{\"archive_size_bytes\":6440615330,\"id\":\"170\",\"kind\":\"VM\",\"logical_id\":\"vm170\"},{\"archive_size_bytes\":2022982219,\"id\":\"171\",\"kind\":\"VM\",\"logical_id\":\"vm171\"},{\"archive_size_bytes\":1557482946,\"id\":\"190\",\"kind\":\"VM\",\"logical_id\":\"vm190\"},{\"archive_size_bytes\":10998362942,\"id\":\"9130\",\"kind\":\"VM\",\"logical_id\":\"vm9130\"},{\"archive_size_bytes\":3182,\"id\":\"filebrowser\",\"kind\":\"APP\",\"logical_id\":\"app-filebrowser\"},{\"archive_size_bytes\":156203,\"id\":\"skladchik-reports-monitor\",\"kind\":\"APP\",\"logical_id\":\"app-skladchik-reports-monitor\"},{\"archive_size_bytes\":270193546,\"id\":\"immich\",\"kind\":\"APP\",\"logical_id\":\"app-immich\"},{\"archive_size_bytes\":75522547,\"id\":\"hbarhub\",\"kind\":\"APP\",\"logical_id\":\"app-hbarhub\"},{\"archive_size_bytes\":8975409,\"id\":\"control-bootstrap-v2\",\"kind\":\"CONTROL\",\"logical_id\":\"control-bootstrap-v2\"}]}\n{\"clouds\":[{\"free_bytes\":786104095252,\"remote\":\"pve01-mail-01-crypt\",\"total_bytes\":1099511627776,\"used_bytes\":313407532524},{\"free_bytes\":786104031393,\"remote\":\"pve01-mail-02-crypt\",\"total_bytes\":1099511627776,\"used_bytes\":313407596383}],\"observed_at_epoch\":1788595592,\"schema\":\"homelab-backup-observability-snapshot-v1\",\"targets\":[{\"archive_size_bytes\":478357303,\"id\":\"110\",\"kind\":\"VM\",\"logical_id\":\"vm110\"},{\"archive_size_bytes\":369892675,\"id\":\"111\",\"kind\":\"VM\",\"logical_id\":\"vm111\"},{\"archive_size_bytes\":223672749,\"id\":\"112\",\"kind\":\"VM\",\"logical_id\":\"vm112\"},{\"archive_size_bytes\":221529138,\"id\":\"113\",\"kind\":\"VM\",\"logical_id\":\"vm113\"},{\"archive_size_bytes\":10614525791,\"id\":\"130\",\"kind\":\"VM\",\"logical_id\":\"vm130\"},{\"archive_size_bytes\":887338816,\"id\":\"150\",\"kind\":\"VM\",\"logical_id\":\"vm150\"},{\"archive_size_bytes\":3615630758,\"id\":\"160\",\"kind\":\"VM\",\"logical_id\":\"vm160\"},{\"archive_size_bytes\":6440615330,\"id\":\"170\",\"kind\":\"VM\",\"logical_id\":\"vm170\"},{\"archive_size_bytes\":2022982219,\"id\":\"171\",\"kind\":\"VM\",\"logical_id\":\"vm171\"},{\"archive_size_bytes\":1557482946,\"id\":\"190\",\"kind\":\"VM\",\"logical_id\":\"vm190\"},{\"archive_size_bytes\":10998362942,\"id\":\"9130\",\"kind\":\"VM\",\"logical_id\":\"vm9130\"},{\"archive_size_bytes\":3182,\"id\":\"filebrowser\",\"kind\":\"APP\",\"logical_id\":\"app-filebrowser\"},{\"archive_size_bytes\":156203,\"id\":\"skladchik-reports-monitor\",\"kind\":\"APP\",\"logical_id\":\"app-skladchik-reports-monitor\"},{\"archive_size_bytes\":270193546,\"id\":\"immich\",\"kind\":\"APP\",\"logical_id\":\"app-immich\"},{\"archive_size_bytes\":75522547,\"id\":\"hbarhub\",\"kind\":\"APP\",\"logical_id\":\"app-hbarhub\"},{\"archive_size_bytes\":8975409,\"id\":\"control-bootstrap-v2\",\"kind\":\"CONTROL\",\"logical_id\":\"control-bootstrap-v2\"}]}\n{\"clouds\":[{\"free_bytes\":767079957393,\"remote\":\"pve01-mail-01-crypt\",\"total_bytes\":1099511627776,\"used_bytes\":332431670383},{\"free_bytes\":768690899582,\"remote\":\"pve01-mail-02-crypt\",\"total_bytes\":1099511627776,\"used_bytes\":330820728194}],\"observed_at_epoch\":1788600970,\"schema\":\"homelab-backup-observability-snapshot-v1\",\"targets\":[{\"archive_size_bytes\":478357303,\"id\":\"110\",\"kind\":\"VM\",\"logical_id\":\"vm110\"},{\"archive_size_bytes\":369892675,\"id\":\"111\",\"kind\":\"VM\",\"logical_id\":\"vm111\"},{\"archive_size_bytes\":223672749,\"id\":\"112\",\"kind\":\"VM\",\"logical_id\":\"vm112\"},{\"archive_size_bytes\":221529138,\"id\":\"113\",\"kind\":\"VM\",\"logical_id\":\"vm113\"},{\"archive_size_bytes\":10591109796,\"id\":\"130\",\"kind\":\"VM\",\"logical_id\":\"vm130\"},{\"archive_size_bytes\":887338816,\"id\":\"150\",\"kind\":\"VM\",\"logical_id\":\"vm150\"},{\"archive_size_bytes\":3615630758,\"id\":\"160\",\"kind\":\"VM\",\"logical_id\":\"vm160\"},{\"archive_size_bytes\":6471807402,\"id\":\"170\",\"kind\":\"VM\",\"logical_id\":\"vm170\"},{\"archive_size_bytes\":1557482946,\"id\":\"190\",\"kind\":\"VM\",\"logical_id\":\"vm190\"},{\"archive_size_bytes\":10998362942,\"id\":\"9130\",\"kind\":\"VM\",\"logical_id\":\"vm9130\"},{\"archive_size_bytes\":3189,\"id\":\"filebrowser\",\"kind\":\"APP\",\"logical_id\":\"app-filebrowser\"},{\"archive_size_bytes\":156634,\"id\":\"skladchik-reports-monitor\",\"kind\":\"APP\",\"logical_id\":\"app-skladchik-reports-monitor\"},{\"archive_size_bytes\":270193548,\"id\":\"immich\",\"kind\":\"APP\",\"logical_id\":\"app-immich\"},{\"archive_size_bytes\":75598903,\"id\":\"hbarhub\",\"kind\":\"APP\",\"logical_id\":\"app-hbarhub\"},{\"archive_size_bytes\":8975409,\"id\":\"control-bootstrap-v2\",\"kind\":\"CONTROL\",\"logical_id\":\"control-bootstrap-v2\"}]}\n{\"clouds\":[{\"free_bytes\":763045869314,\"remote\":\"pve01-mail-01-crypt\",\"total_bytes\":1099511627776,\"used_bytes\":336465758462},{\"free_bytes\":763045805455,\"remote\":\"pve01-mail-02-crypt\",\"total_bytes\":1099511627776,\"used_bytes\":336465822321}],\"observed_at_epoch\":1788604613,\"schema\":\"homelab-backup-observability-snapshot-v1\",\"targets\":[{\"archive_size_bytes\":478357303,\"id\":\"110\",\"kind\":\"VM\",\"logical_id\":\"vm110\"},{\"archive_size_bytes\":369892675,\"id\":\"111\",\"kind\":\"VM\",\"logical_id\":\"vm111\"},{\"archive_size_bytes\":223672749,\"id\":\"112\",\"kind\":\"VM\",\"logical_id\":\"vm112\"},{\"archive_size_bytes\":221529138,\"id\":\"113\",\"kind\":\"VM\",\"logical_id\":\"vm113\"},{\"archive_size_bytes\":10591109796,\"id\":\"130\",\"kind\":\"VM\",\"logical_id\":\"vm130\"},{\"archive_size_bytes\":887338816,\"id\":\"150\",\"kind\":\"VM\",\"logical_id\":\"vm150\"},{\"archive_size_bytes\":6471807402,\"id\":\"170\",\"kind\":\"VM\",\"logical_id\":\"vm170\"},{\"archive_size_bytes\":2023943350,\"id\":\"171\",\"kind\":\"VM\",\"logical_id\":\"vm171\"},{\"archive_size_bytes\":1557482946,\"id\":\"190\",\"kind\":\"VM\",\"logical_id\":\"vm190\"},{\"archive_size_bytes\":10998362942,\"id\":\"9130\",\"kind\":\"VM\",\"logical_id\":\"vm9130\"},{\"archive_size_bytes\":3189,\"id\":\"filebrowser\",\"kind\":\"APP\",\"logical_id\":\"app-filebrowser\"},{\"archive_size_bytes\":156634,\"id\":\"skladchik-reports-monitor\",\"kind\":\"APP\",\"logical_id\":\"app-skladchik-reports-monitor\"},{\"archive_size_bytes\":270193548,\"id\":\"immich\",\"kind\":\"APP\",\"logical_id\":\"app-immich\"},{\"archive_size_bytes\":75598903,\"id\":\"hbarhub\",\"kind\":\"APP\",\"logical_id\":\"app-hbarhub\"},{\"archive_size_bytes\":8975409,\"id\":\"control-bootstrap-v2\",\"kind\":\"CONTROL\",\"logical_id\":\"control-bootstrap-v2\"}]}\n{\"clouds\":[{\"free_bytes\":763036886053,\"remote\":\"pve01-mail-01-crypt\",\"total_bytes\":1099511627776,\"used_bytes\":336474741723},{\"free_bytes\":763036822194,\"remote\":\"pve01-mail-02-crypt\",\"total_bytes\":1099511627776,\"used_bytes\":336474805582}],\"observed_at_epoch\":1788609912,\"schema\":\"homelab-backup-observability-snapshot-v1\",\"targets\":[{\"archive_size_bytes\":478357303,\"id\":\"110\",\"kind\":\"VM\",\"logical_id\":\"vm110\"},{\"archive_size_bytes\":369892675,\"id\":\"111\",\"kind\":\"VM\",\"logical_id\":\"vm111\"},{\"archive_size_bytes\":223672749,\"id\":\"112\",\"kind\":\"VM\",\"logical_id\":\"vm112\"},{\"archive_size_bytes\":221529138,\"id\":\"113\",\"kind\":\"VM\",\"logical_id\":\"vm113\"},{\"archive_size_bytes\":10591109796,\"id\":\"130\",\"kind\":\"VM\",\"logical_id\":\"vm130\"},{\"archive_size_bytes\":887338816,\"id\":\"150\",\"kind\":\"VM\",\"logical_id\":\"vm150\"},{\"archive_size_bytes\":3619770210,\"id\":\"160\",\"kind\":\"VM\",\"logical_id\":\"vm160\"},{\"archive_size_bytes\":6471807402,\"id\":\"170\",\"kind\":\"VM\",\"logical_id\":\"vm170\"},{\"archive_size_bytes\":2023943350,\"id\":\"171\",\"kind\":\"VM\",\"logical_id\":\"vm171\"},{\"archive_size_bytes\":1557482946,\"id\":\"190\",\"kind\":\"VM\",\"logical_id\":\"vm190\"},{\"archive_size_bytes\":10998362942,\"id\":\"9130\",\"kind\":\"VM\",\"logical_id\":\"vm9130\"},{\"archive_size_bytes\":3189,\"id\":\"filebrowser\",\"kind\":\"APP\",\"logical_id\":\"app-filebrowser\"},{\"archive_size_bytes\":156634,\"id\":\"skladchik-reports-monitor\",\"kind\":\"APP\",\"logical_id\":\"app-skladchik-reports-monitor\"},{\"archive_size_bytes\":270193548,\"id\":\"immich\",\"kind\":\"APP\",\"logical_id\":\"app-immich\"},{\"archive_size_bytes\":75598903,\"id\":\"hbarhub\",\"kind\":\"APP\",\"logical_id\":\"app-hbarhub\"},{\"archive_size_bytes\":8978589,\"id\":\"control-bootstrap-v2\",\"kind\":\"CONTROL\",\"logical_id\":\"control-bootstrap-v2\"}]}\n{\"clouds\":[{\"free_bytes\":763036886053,\"remote\":\"pve01-mail-01-crypt\",\"total_bytes\":1099511627776,\"used_bytes\":336474741723},{\"free_bytes\":763036822194,\"remote\":\"pve01-mail-02-crypt\",\"total_bytes\":1099511627776,\"used_bytes\":336474805582}],\"observed_at_epoch\":1788613592,\"schema\":\"homelab-backup-observability-snapshot-v1\",\"targets\":[{\"archive_size_bytes\":478357303,\"id\":\"110\",\"kind\":\"VM\",\"logical_id\":\"vm110\"},{\"archive_size_bytes\":369892675,\"id\":\"111\",\"kind\":\"VM\",\"logical_id\":\"vm111\"},{\"archive_size_bytes\":223672749,\"id\":\"112\",\"kind\":\"VM\",\"logical_id\":\"vm112\"},{\"archive_size_bytes\":221529138,\"id\":\"113\",\"kind\":\"VM\",\"logical_id\":\"vm113\"},{\"archive_size_bytes\":10591109796,\"id\":\"130\",\"kind\":\"VM\",\"logical_id\":\"vm130\"},{\"archive_size_bytes\":887338816,\"id\":\"150\",\"kind\":\"VM\",\"logical_id\":\"vm150\"},{\"archive_size_bytes\":3619770210,\"id\":\"160\",\"kind\":\"VM\",\"logical_id\":\"vm160\"},{\"archive_size_bytes\":6471807402,\"id\":\"170\",\"kind\":\"VM\",\"logical_id\":\"vm170\"},{\"archive_size_bytes\":2023943350,\"id\":\"171\",\"kind\":\"VM\",\"logical_id\":\"vm171\"},{\"archive_size_bytes\":1557482946,\"id\":\"190\",\"kind\":\"VM\",\"logical_id\":\"vm190\"},{\"archive_size_bytes\":10998362942,\"id\":\"9130\",\"kind\":\"VM\",\"logical_id\":\"vm9130\"},{\"archive_size_bytes\":3189,\"id\":\"filebrowser\",\"kind\":\"APP\",\"logical_id\":\"app-filebrowser\"},{\"archive_size_bytes\":156634,\"id\":\"skladchik-reports-monitor\",\"kind\":\"APP\",\"logical_id\":\"app-skladchik-reports-monitor\"},{\"archive_size_bytes\":270193548,\"id\":\"immich\",\"kind\":\"APP\",\"logical_id\":\"app-immich\"},{\"archive_size_bytes\":75598903,\"id\":\"hbarhub\",\"kind\":\"APP\",\"logical_id\":\"app-hbarhub\"},{\"archive_size_bytes\":8978589,\"id\":\"control-bootstrap-v2\",\"kind\":\"CONTROL\",\"logical_id\":\"control-bootstrap-v2\"}]}\n{\"clouds\":[{\"free_bytes\":763036886053,\"remote\":\"pve01-mail-01-crypt\",\"total_bytes\":1099511627776,\"used_bytes\":336474741723},{\"free_bytes\":763036822194,\"remote\":\"pve01-mail-02-crypt\",\"total_bytes\":1099511627776,\"used_bytes\":336474805582}],\"observed_at_epoch\":1788618970,\"schema\":\"homelab-backup-observability-snapshot-v1\",\"targets\":[{\"archive_size_bytes\":478357303,\"id\":\"110\",\"kind\":\"VM\",\"logical_id\":\"vm110\"},{\"archive_size_bytes\":369892675,\"id\":\"111\",\"kind\":\"VM\",\"logical_id\":\"vm111\"},{\"archive_size_bytes\":223672749,\"id\":\"112\",\"kind\":\"VM\",\"logical_id\":\"vm112\"},{\"archive_size_bytes\":221529138,\"id\":\"113\",\"kind\":\"VM\",\"logical_id\":\"vm113\"},{\"archive_size_bytes\":10591109796,\"id\":\"130\",\"kind\":\"VM\",\"logical_id\":\"vm130\"},{\"archive_size_bytes\":887338816,\"id\":\"150\",\"kind\":\"VM\",\"logical_id\":\"vm150\"},{\"archive_size_bytes\":3619770210,\"id\":\"160\",\"kind\":\"VM\",\"logical_id\":\"vm160\"},{\"archive_size_bytes\":6471807402,\"id\":\"170\",\"kind\":\"VM\",\"logical_id\":\"vm170\"},{\"archive_size_bytes\":2023943350,\"id\":\"171\",\"kind\":\"VM\",\"logical_id\":\"vm171\"},{\"archive_size_bytes\":1557482946,\"id\":\"190\",\"kind\":\"VM\",\"logical_id\":\"vm190\"},{\"archive_size_bytes\":10998362942,\"id\":\"9130\",\"kind\":\"VM\",\"logical_id\":\"vm9130\"},{\"archive_size_bytes\":3189,\"id\":\"filebrowser\",\"kind\":\"APP\",\"logical_id\":\"app-filebrowser\"},{\"archive_size_bytes\":156634,\"id\":\"skladchik-reports-monitor\",\"kind\":\"APP\",\"logical_id\":\"app-skladchik-reports-monitor\"},{\"archive_size_bytes\":270193548,\"id\":\"immich\",\"kind\":\"APP\",\"logical_id\":\"app-immich\"},{\"archive_size_bytes\":75598903,\"id\":\"hbarhub\",\"kind\":\"APP\",\"logical_id\":\"app-hbarhub\"},{\"archive_size_bytes\":8978589,\"id\":\"control-bootstrap-v2\",\"kind\":\"CONTROL\",\"logical_id\":\"control-bootstrap-v2\"}]}\n{\"clouds\":[{\"free_bytes\":763036886053,\"remote\":\"pve01-mail-01-crypt\",\"total_bytes\":1099511627776,\"used_bytes\":336474741723},{\"free_bytes\":763036822194,\"remote\":\"pve01-mail-02-crypt\",\"total_bytes\":1099511627776,\"used_bytes\":336474805582}],\"observed_at_epoch\":1788624358,\"schema\":\"homelab-backup-observability-snapshot-v1\",\"targets\":[{\"archive_size_bytes\":478357303,\"id\":\"110\",\"kind\":\"VM\",\"logical_id\":\"vm110\"},{\"archive_size_bytes\":369892675,\"id\":\"111\",\"kind\":\"VM\",\"logical_id\":\"vm111\"},{\"archive_size_bytes\":223672749,\"id\":\"112\",\"kind\":\"VM\",\"logical_id\":\"vm112\"},{\"archive_size_bytes\":221529138,\"id\":\"113\",\"kind\":\"VM\",\"logical_id\":\"vm113\"},{\"archive_size_bytes\":10591109796,\"id\":\"130\",\"kind\":\"VM\",\"logical_id\":\"vm130\"},{\"archive_size_bytes\":887338816,\"id\":\"150\",\"kind\":\"VM\",\"logical_id\":\"vm150\"},{\"archive_size_bytes\":3619770210,\"id\":\"160\",\"kind\":\"VM\",\"logical_id\":\"vm160\"},{\"archive_size_bytes\":6471807402,\"id\":\"170\",\"kind\":\"VM\",\"logical_id\":\"vm170\"},{\"archive_size_bytes\":2023943350,\"id\":\"171\",\"kind\":\"VM\",\"logical_id\":\"vm171\"},{\"archive_size_bytes\":1557482946,\"id\":\"190\",\"kind\":\"VM\",\"logical_id\":\"vm190\"},{\"archive_size_bytes\":10998362942,\"id\":\"9130\",\"kind\":\"VM\",\"logical_id\":\"vm9130\"},{\"archive_size_bytes\":3189,\"id\":\"filebrowser\",\"kind\":\"APP\",\"logical_id\":\"app-filebrowser\"},{\"archive_size_bytes\":156634,\"id\":\"skladchik-reports-monitor\",\"kind\":\"APP\",\"logical_id\":\"app-skladchik-reports-monitor\"},{\"archive_size_bytes\":270193548,\"id\":\"immich\",\"kind\":\"APP\",\"logical_id\":\"app-immich\"},{\"archive_size_bytes\":75598903,\"id\":\"hbarhub\",\"kind\":\"APP\",\"logical_id\":\"app-hbarhub\"},{\"archive_size_bytes\":8978589,\"id\":\"control-bootstrap-v2\",\"kind\":\"CONTROL\",\"logical_id\":\"control-bootstrap-v2\"}]}\n{\"clouds\":[{\"free_bytes\":763036886053,\"remote\":\"pve01-mail-01-crypt\",\"total_bytes\":1099511627776,\"used_bytes\":336474741723},{\"free_bytes\":763036822194,\"remote\":\"pve01-mail-02-crypt\",\"total_bytes\":1099511627776,\"used_bytes\":336474805582}],\"observed_at_epoch\":1788628013,\"schema\":\"homelab-backup-observability-snapshot-v1\",\"targets\":[{\"archive_size_bytes\":478357303,\"id\":\"110\",\"kind\":\"VM\",\"logical_id\":\"vm110\"},{\"archive_size_bytes\":369892675,\"id\":\"111\",\"kind\":\"VM\",\"logical_id\":\"vm111\"},{\"archive_size_bytes\":223672749,\"id\":\"112\",\"kind\":\"VM\",\"logical_id\":\"vm112\"},{\"archive_size_bytes\":221529138,\"id\":\"113\",\"kind\":\"VM\",\"logical_id\":\"vm113\"},{\"archive_size_bytes\":10591109796,\"id\":\"130\",\"kind\":\"VM\",\"logical_id\":\"vm130\"},{\"archive_size_bytes\":887338816,\"id\":\"150\",\"kind\":\"VM\",\"logical_id\":\"vm150\"},{\"archive_size_bytes\":3619770210,\"id\":\"160\",\"kind\":\"VM\",\"logical_id\":\"vm160\"},{\"archive_size_bytes\":6471807402,\"id\":\"170\",\"kind\":\"VM\",\"logical_id\":\"vm170\"},{\"archive_size_bytes\":2023943350,\"id\":\"171\",\"kind\":\"VM\",\"logical_id\":\"vm171\"},{\"archive_size_bytes\":1557482946,\"id\":\"190\",\"kind\":\"VM\",\"logical_id\":\"vm190\"},{\"archive_size_bytes\":10998362942,\"id\":\"9130\",\"kind\":\"VM\",\"logical_id\":\"vm9130\"},{\"archive_size_bytes\":3189,\"id\":\"filebrowser\",\"kind\":\"APP\",\"logical_id\":\"app-filebrowser\"},{\"archive_size_bytes\":156634,\"id\":\"skladchik-reports-monitor\",\"kind\":\"APP\",\"logical_id\":\"app-skladchik-reports-monitor\"},{\"archive_size_bytes\":270193548,\"id\":\"immich\",\"kind\":\"APP\",\"logical_id\":\"app-immich\"},{\"archive_size_bytes\":75598903,\"id\":\"hbarhub\",\"kind\":\"APP\",\"logical_id\":\"app-hbarhub\"},{\"archive_size_bytes\":8978589,\"id\":\"control-bootstrap-v2\",\"kind\":\"CONTROL\",\"logical_id\":\"control-bootstrap-v2\"}]}\n{\"clouds\":[{\"free_bytes\":763036886053,\"remote\":\"pve01-mail-01-crypt\",\"total_bytes\":1099511627776,\"used_bytes\":336474741723},{\"free_bytes\":763036822194,\"remote\":\"pve01-mail-02-crypt\",\"total_bytes\":1099511627776,\"used_bytes\":336474805582}],\"observed_at_epoch\":1788633392,\"schema\":\"homelab-backup-observability-snapshot-v1\",\"targets\":[{\"archive_size_bytes\":478357303,\"id\":\"110\",\"kind\":\"VM\",\"logical_id\":\"vm110\"},{\"archive_size_bytes\":369892675,\"id\":\"111\",\"kind\":\"VM\",\"logical_id\":\"vm111\"},{\"archive_size_bytes\":223672749,\"id\":\"112\",\"kind\":\"VM\",\"logical_id\":\"vm112\"},{\"archive_size_bytes\":221529138,\"id\":\"113\",\"kind\":\"VM\",\"logical_id\":\"vm113\"},{\"archive_size_bytes\":10591109796,\"id\":\"130\",\"kind\":\"VM\",\"logical_id\":\"vm130\"},{\"archive_size_bytes\":887338816,\"id\":\"150\",\"kind\":\"VM\",\"logical_id\":\"vm150\"},{\"archive_size_bytes\":3619770210,\"id\":\"160\",\"kind\":\"VM\",\"logical_id\":\"vm160\"},{\"archive_size_bytes\":6471807402,\"id\":\"170\",\"kind\":\"VM\",\"logical_id\":\"vm170\"},{\"archive_size_bytes\":2023943350,\"id\":\"171\",\"kind\":\"VM\",\"logical_id\":\"vm171\"},{\"archive_size_bytes\":1557482946,\"id\":\"190\",\"kind\":\"VM\",\"logical_id\":\"vm190\"},{\"archive_size_bytes\":10998362942,\"id\":\"9130\",\"kind\":\"VM\",\"logical_id\":\"vm9130\"},{\"archive_size_bytes\":3189,\"id\":\"filebrowser\",\"kind\":\"APP\",\"logical_id\":\"app-filebrowser\"},{\"archive_size_bytes\":156634,\"id\":\"skladchik-reports-monitor\",\"kind\":\"APP\",\"logical_id\":\"app-skladchik-reports-monitor\"},{\"archive_size_bytes\":270193548,\"id\":\"immich\",\"kind\":\"APP\",\"logical_id\":\"app-immich\"},{\"archive_size_bytes\":75598903,\"id\":\"hbarhub\",\"kind\":\"APP\",\"logical_id\":\"app-hbarhub\"},{\"archive_size_bytes\":8978589,\"id\":\"control-bootstrap-v2\",\"kind\":\"CONTROL\",\"logical_id\":\"control-bootstrap-v2\"}]}\n{\"clouds\":[{\"free_bytes\":763036886053,\"remote\":\"pve01-mail-01-crypt\",\"total_bytes\":1099511627776,\"used_bytes\":336474741723},{\"free_bytes\":763036822194,\"remote\":\"pve01-mail-02-crypt\",\"total_bytes\":1099511627776,\"used_bytes\":336474805582}],\"observed_at_epoch\":1788638772,\"schema\":\"homelab-backup-observability-snapshot-v1\",\"targets\":[{\"archive_size_bytes\":478357303,\"id\":\"110\",\"kind\":\"VM\",\"logical_id\":\"vm110\"},{\"archive_size_bytes\":369892675,\"id\":\"111\",\"kind\":\"VM\",\"logical_id\":\"vm111\"},{\"archive_size_bytes\":223672749,\"id\":\"112\",\"kind\":\"VM\",\"logical_id\":\"vm112\"},{\"archive_size_bytes\":221529138,\"id\":\"113\",\"kind\":\"VM\",\"logical_id\":\"vm113\"},{\"archive_size_bytes\":10591109796,\"id\":\"130\",\"kind\":\"VM\",\"logical_id\":\"vm130\"},{\"archive_size_bytes\":887338816,\"id\":\"150\",\"kind\":\"VM\",\"logical_id\":\"vm150\"},{\"archive_size_bytes\":3619770210,\"id\":\"160\",\"kind\":\"VM\",\"logical_id\":\"vm160\"},{\"archive_size_bytes\":6471807402,\"id\":\"170\",\"kind\":\"VM\",\"logical_id\":\"vm170\"},{\"archive_size_bytes\":2023943350,\"id\":\"171\",\"kind\":\"VM\",\"logical_id\":\"vm171\"},{\"archive_size_bytes\":1557482946,\"id\":\"190\",\"kind\":\"VM\",\"logical_id\":\"vm190\"},{\"archive_size_bytes\":10998362942,\"id\":\"9130\",\"kind\":\"VM\",\"logical_id\":\"vm9130\"},{\"archive_size_bytes\":3189,\"id\":\"filebrowser\",\"kind\":\"APP\",\"logical_id\":\"app-filebrowser\"},{\"archive_size_bytes\":156634,\"id\":\"skladchik-reports-monitor\",\"kind\":\"APP\",\"logical_id\":\"app-skladchik-reports-monitor\"},{\"archive_size_bytes\":270193548,\"id\":\"immich\",\"kind\":\"APP\",\"logical_id\":\"app-immich\"},{\"archive_size_bytes\":75598903,\"id\":\"hbarhub\",\"kind\":\"APP\",\"logical_id\":\"app-hbarhub\"},{\"archive_size_bytes\":8978589,\"id\":\"control-bootstrap-v2\",\"kind\":\"CONTROL\",\"logical_id\":\"control-bootstrap-v2\"}]}\n{\"clouds\":[{\"free_bytes\":756548494504,\"remote\":\"pve01-mail-01-crypt\",\"total_bytes\":1099511627776,\"used_bytes\":342963133272},{\"free_bytes\":763036822194,\"remote\":\"pve01-mail-02-crypt\",\"total_bytes\":1099511627776,\"used_bytes\":336474805582}],\"observed_at_epoch\":1788644170,\"schema\":\"homelab-backup-observability-snapshot-v1\",\"targets\":[{\"archive_size_bytes\":478357303,\"id\":\"110\",\"kind\":\"VM\",\"logical_id\":\"vm110\"},{\"archive_size_bytes\":369892675,\"id\":\"111\",\"kind\":\"VM\",\"logical_id\":\"vm111\"},{\"archive_size_bytes\":223672749,\"id\":\"112\",\"kind\":\"VM\",\"logical_id\":\"vm112\"},{\"archive_size_bytes\":221529138,\"id\":\"113\",\"kind\":\"VM\",\"logical_id\":\"vm113\"},{\"archive_size_bytes\":10591109796,\"id\":\"130\",\"kind\":\"VM\",\"logical_id\":\"vm130\"},{\"archive_size_bytes\":887338816,\"id\":\"150\",\"kind\":\"VM\",\"logical_id\":\"vm150\"},{\"archive_size_bytes\":3619770210,\"id\":\"160\",\"kind\":\"VM\",\"logical_id\":\"vm160\"},{\"archive_size_bytes\":2023943350,\"id\":\"171\",\"kind\":\"VM\",\"logical_id\":\"vm171\"},{\"archive_size_bytes\":1557482946,\"id\":\"190\",\"kind\":\"VM\",\"logical_id\":\"vm190\"},{\"archive_size_bytes\":10998362942,\"id\":\"9130\",\"kind\":\"VM\",\"logical_id\":\"vm9130\"},{\"archive_size_bytes\":3189,\"id\":\"filebrowser\",\"kind\":\"APP\",\"logical_id\":\"app-filebrowser\"},{\"archive_size_bytes\":156634,\"id\":\"skladchik-reports-monitor\",\"kind\":\"APP\",\"logical_id\":\"app-skladchik-reports-monitor\"},{\"archive_size_bytes\":270193548,\"id\":\"immich\",\"kind\":\"APP\",\"logical_id\":\"app-immich\"},{\"archive_size_bytes\":75598903,\"id\":\"hbarhub\",\"kind\":\"APP\",\"logical_id\":\"app-hbarhub\"},{\"archive_size_bytes\":8978589,\"id\":\"control-bootstrap-v2\",\"kind\":\"CONTROL\",\"logical_id\":\"control-bootstrap-v2\"}]}\n{\"clouds\":[{\"free_bytes\":743574336966,\"remote\":\"pve01-mail-01-crypt\",\"total_bytes\":1099511627776,\"used_bytes\":355937290810},{\"free_bytes\":743574273107,\"remote\":\"pve01-mail-02-crypt\",\"total_bytes\":1099511627776,\"used_bytes\":355937354669}],\"observed_at_epoch\":1788649558,\"schema\":\"homelab-backup-observability-snapshot-v1\",\"targets\":[{\"archive_size_bytes\":478357303,\"id\":\"110\",\"kind\":\"VM\",\"logical_id\":\"vm110\"},{\"archive_size_bytes\":369892675,\"id\":\"111\",\"kind\":\"VM\",\"logical_id\":\"vm111\"},{\"archive_size_bytes\":223672749,\"id\":\"112\",\"kind\":\"VM\",\"logical_id\":\"vm112\"},{\"archive_size_bytes\":221529138,\"id\":\"113\",\"kind\":\"VM\",\"logical_id\":\"vm113\"},{\"archive_size_bytes\":10588466039,\"id\":\"130\",\"kind\":\"VM\",\"logical_id\":\"vm130\"},{\"archive_size_bytes\":887338816,\"id\":\"150\",\"kind\":\"VM\",\"logical_id\":\"vm150\"},{\"archive_size_bytes\":3619770210,\"id\":\"160\",\"kind\":\"VM\",\"logical_id\":\"vm160\"},{\"archive_size_bytes\":6486807437,\"id\":\"170\",\"kind\":\"VM\",\"logical_id\":\"vm170\"},{\"archive_size_bytes\":2027528282,\"id\":\"171\",\"kind\":\"VM\",\"logical_id\":\"vm171\"},{\"archive_size_bytes\":1557482946,\"id\":\"190\",\"kind\":\"VM\",\"logical_id\":\"vm190\"},{\"archive_size_bytes\":10998362942,\"id\":\"9130\",\"kind\":\"VM\",\"logical_id\":\"vm9130\"},{\"archive_size_bytes\":3191,\"id\":\"filebrowser\",\"kind\":\"APP\",\"logical_id\":\"app-filebrowser\"},{\"archive_size_bytes\":157473,\"id\":\"skladchik-reports-monitor\",\"kind\":\"APP\",\"logical_id\":\"app-skladchik-reports-monitor\"},{\"archive_size_bytes\":270193543,\"id\":\"immich\",\"kind\":\"APP\",\"logical_id\":\"app-immich\"},{\"archive_size_bytes\":75669106,\"id\":\"hbarhub\",\"kind\":\"APP\",\"logical_id\":\"app-hbarhub\"},{\"archive_size_bytes\":8958389,\"id\":\"control-bootstrap-v2\",\"kind\":\"CONTROL\",\"logical_id\":\"control-bootstrap-v2\"}]}\n{\"clouds\":[{\"free_bytes\":743574336966,\"remote\":\"pve01-mail-01-crypt\",\"total_bytes\":1099511627776,\"used_bytes\":355937290810},{\"free_bytes\":743574273107,\"remote\":\"pve01-mail-02-crypt\",\"total_bytes\":1099511627776,\"used_bytes\":355937354669}],\"observed_at_epoch\":1788653194,\"schema\":\"homelab-backup-observability-snapshot-v1\",\"targets\":[{\"archive_size_bytes\":478357303,\"id\":\"110\",\"kind\":\"VM\",\"logical_id\":\"vm110\"},{\"archive_size_bytes\":369892675,\"id\":\"111\",\"kind\":\"VM\",\"logical_id\":\"vm111\"},{\"archive_size_bytes\":223672749,\"id\":\"112\",\"kind\":\"VM\",\"logical_id\":\"vm112\"},{\"archive_size_bytes\":221529138,\"id\":\"113\",\"kind\":\"VM\",\"logical_id\":\"vm113\"},{\"archive_size_bytes\":10588466039,\"id\":\"130\",\"kind\":\"VM\",\"logical_id\":\"vm130\"},{\"archive_size_bytes\":887338816,\"id\":\"150\",\"kind\":\"VM\",\"logical_id\":\"vm150\"},{\"archive_size_bytes\":3619770210,\"id\":\"160\",\"kind\":\"VM\",\"logical_id\":\"vm160\"},{\"archive_size_bytes\":6486807437,\"id\":\"170\",\"kind\":\"VM\",\"logical_id\":\"vm170\"},{\"archive_size_bytes\":2027528282,\"id\":\"171\",\"kind\":\"VM\",\"logical_id\":\"vm171\"},{\"archive_size_bytes\":1557482946,\"id\":\"190\",\"kind\":\"VM\",\"logical_id\":\"vm190\"},{\"archive_size_bytes\":10998362942,\"id\":\"9130\",\"kind\":\"VM\",\"logical_id\":\"vm9130\"},{\"archive_size_bytes\":3191,\"id\":\"filebrowser\",\"kind\":\"APP\",\"logical_id\":\"app-filebrowser\"},{\"archive_size_bytes\":157473,\"id\":\"skladchik-reports-monitor\",\"kind\":\"APP\",\"logical_id\":\"app-skladchik-reports-monitor\"},{\"archive_size_bytes\":270193543,\"id\":\"immich\",\"kind\":\"APP\",\"logical_id\":\"app-immich\"},{\"archive_size_bytes\":75669106,\"id\":\"hbarhub\",\"kind\":\"APP\",\"logical_id\":\"app-hbarhub\"},{\"archive_size_bytes\":8958389,\"id\":\"control-bootstrap-v2\",\"kind\":\"CONTROL\",\"logical_id\":\"control-bootstrap-v2\"}]}\n{\"clouds\":[{\"free_bytes\":739949922534,\"remote\":\"pve01-mail-01-crypt\",\"total_bytes\":1099511627776,\"used_bytes\":359561705242},{\"free_bytes\":739949858675,\"remote\":\"pve01-mail-02-crypt\",\"total_bytes\":1099511627776,\"used_bytes\":359561769101}],\"observed_at_epoch\":1788656802,\"schema\":\"homelab-backup-observability-snapshot-v1\",\"targets\":[{\"archive_size_bytes\":478357303,\"id\":\"110\",\"kind\":\"VM\",\"logical_id\":\"vm110\"},{\"archive_size_bytes\":369892675,\"id\":\"111\",\"kind\":\"VM\",\"logical_id\":\"vm111\"},{\"archive_size_bytes\":223672749,\"id\":\"112\",\"kind\":\"VM\",\"logical_id\":\"vm112\"},{\"archive_size_bytes\":221529138,\"id\":\"113\",\"kind\":\"VM\",\"logical_id\":\"vm113\"},{\"archive_size_bytes\":10588466039,\"id\":\"130\",\"kind\":\"VM\",\"logical_id\":\"vm130\"},{\"archive_size_bytes\":887338816,\"id\":\"150\",\"kind\":\"VM\",\"logical_id\":\"vm150\"},{\"archive_size_bytes\":3623527665,\"id\":\"160\",\"kind\":\"VM\",\"logical_id\":\"vm160\"},{\"archive_size_bytes\":6486807437,\"id\":\"170\",\"kind\":\"VM\",\"logical_id\":\"vm170\"},{\"archive_size_bytes\":2027528282,\"id\":\"171\",\"kind\":\"VM\",\"logical_id\":\"vm171\"},{\"archive_size_bytes\":1557482946,\"id\":\"190\",\"kind\":\"VM\",\"logical_id\":\"vm190\"},{\"archive_size_bytes\":10998362942,\"id\":\"9130\",\"kind\":\"VM\",\"logical_id\":\"vm9130\"},{\"archive_size_bytes\":3191,\"id\":\"filebrowser\",\"kind\":\"APP\",\"logical_id\":\"app-filebrowser\"},{\"archive_size_bytes\":157473,\"id\":\"skladchik-reports-monitor\",\"kind\":\"APP\",\"logical_id\":\"app-skladchik-reports-monitor\"},{\"archive_size_bytes\":270193543,\"id\":\"immich\",\"kind\":\"APP\",\"logical_id\":\"app-immich\"},{\"archive_size_bytes\":75669106,\"id\":\"hbarhub\",\"kind\":\"APP\",\"logical_id\":\"app-hbarhub\"},{\"archive_size_bytes\":8958389,\"id\":\"control-bootstrap-v2\",\"kind\":\"CONTROL\",\"logical_id\":\"control-bootstrap-v2\"}]}\nHEALTH_FILE=homebox-backup.txt|mtime=2026-07-09 19:01:20.790193328 +0300|sha256=b388a7150af8169744f923e78e519408710c482afa7c2a3fcc334d1a1d3006e6\nSTATUS=OK TS=2026-07-09T16:01:19Z TYPE=homebox-backup JOB=homelab-pve01-app-homebox-20260709T051713Z SOURCE=latest-appbackup SIZE=8873 LOCAL_BACKUP_DELETED=true JSON=/var/lib/homelab-backup/apps/latest-homebox.json NOTE=service-readiness-backup-proof\nHEALTH_FILE=edge-vm-vzdump-backup.txt|mtime=2026-08-30 21:08:08.621601440 +0300|sha256=35e1b1b24eea8e1ca6506ef437b49606f1e691544f70965db8d30b9b075f9b12\nSTATUS=FAIL\nTS=2026-08-30T18:08:08Z\nTYPE=backup-matrix-generated\nJOB_ID=homelab_edge_vm_vzdump_backup\nJOB_NAME=homelab-edge-vm-offhost-freshness\nMATRIX_MODE=authoritative\nTIMER=homelab-edge-vm-offhost-freshness.timer\nSERVICE=homelab-edge-vm-offhost-freshness.service\nTIMER_ENABLED=not-found\nTIMER_ACTIVE=inactive\nSOURCE_KIND=health_file\nEVIDENCE_LEVEL=artifact_or_health\nSERVICE_RESULT=success\nSERVICE_EXEC_MAIN_STATUS=0\nSOURCE_HEALTH=/var/lib/homelab-health/edge-vm-offhost-freshness.txt\nSOURCE_STATUS=FAIL\nSOURCE_MTIME_UTC=2026-08-08T01:50:13Z\nAGE_SECONDS=1959475\nMAX_AGE_SECONDS=172800\nFRESHNESS=STALE\nLEGACY_SOURCE_PATH=/var/lib/homelab-health/edge-vm-vzdump-backup.txt\nCANONICAL_JOB_ID=homelab_edge_vm_vzdump_backup\nCANONICAL_STATE=/var/lib/homelab-backup-matrix/state.json\nHEALTH_FILE=npmplus-kuma-config-backup.txt|mtime=2026-08-30 21:08:08.440410577 +0300|sha256=be5d4aaace4430857d61615081e8a8ac9fcc577aed6224356cf154114ae08250\nSTATUS=FAIL\nTS=2026-08-30T18:08:08Z\nTYPE=backup-matrix-generated\nJOB_ID=homelab_appbackup_npmplus_edge\nJOB_NAME=homelab-appbackup-npmplus-edge\nMATRIX_MODE=authoritative\nTIMER=homelab-appbackup-npmplus-edge.timer\nSERVICE=homelab-appbackup-npmplus-edge.service\nTIMER_ENABLED=not-found\nTIMER_ACTIVE=inactive\nSOURCE_KIND=health_file\nEVIDENCE_LEVEL=artifact_or_health\nSERVICE_RESULT=success\nSERVICE_EXEC_MAIN_STATUS=0\nSOURCE_HEALTH=/var/lib/homelab-backup-matrix/evidence/homelab_appbackup_npmplus_edge/0-latest-npmplus-edge.json\nSOURCE_STATUS=OK\nSOURCE_MTIME_UTC=2026-08-06T21:21:34Z\nAGE_SECONDS=2061994\nMAX_AGE_SECONDS=172800\nFRESHNESS=STALE\nLEGACY_SOURCE_PATH=/var/lib/homelab-health/npmplus-kuma-config-backup.txt\nCANONICAL_JOB_ID=homelab_appbackup_npmplus_edge\nCANONICAL_STATE=/var/lib/homelab-backup-matrix/state.json\nHEALTH_FILE=backup-coverage-matrix.txt|mtime=2026-08-30 21:08:08.539509156 +0300|sha256=48b3b20a0dc225b93e0f7b8b1c2746649eb6f965d12f996030cc8ec8791604ff\nSTATUS=FAIL\nTS=2026-08-30T18:08:08Z\nTYPE=backup-matrix-generated\nJOB_ID=homelab_backup_coverage_matrix\nJOB_NAME=homelab-backup-coverage-matrix\nMATRIX_MODE=authoritative\nTIMER=homelab-backup-coverage-matrix.timer\nSERVICE=homelab-backup-coverage-matrix.service\nTIMER_ENABLED=not-found\nTIMER_ACTIVE=inactive\nSOURCE_KIND=health_file\nEVIDENCE_LEVEL=artifact_or_health\nSERVICE_RESULT=success\nSERVICE_EXEC_MAIN_STATUS=0\nSOURCE_HEALTH=/var/lib/homelab-backup-matrix/evidence/homelab_backup_coverage_matrix/0-backup-coverage-matrix.txt\nSOURCE_STATUS=OK\nSOURCE_MTIME_UTC=2026-08-06T21:34:03Z\nAGE_SECONDS=2061245\nMAX_AGE_SECONDS=172800\nFRESHNESS=STALE\nLEGACY_SOURCE_PATH=/var/lib/homelab-health/backup-coverage-matrix.txt\nCANONICAL_JOB_ID=homelab_backup_coverage_matrix\nCANONICAL_STATE=/var/lib/homelab-backup-matrix/state.json\nHEALTH_FILE=vm160-remote-backup.txt|mtime=2026-07-21 02:39:14.312087060 +0300|sha256=de5897d501d5bc25ca6fe964bf2643b2f8cac6957f9f577a1ae0cf4c68c3587e\nSTATUS=FAIL\nSERVICE=homelab-vm160-remote-backup\nTIMESTAMP_UTC=20260720T233913Z\nVMID=160\nTARGET_DIR=/mnt/staging/vzdump/vm160-pve02-20260720T233913Z\nRC=1\nHEALTH_FILE=vm-backup.txt|mtime=2026-07-24 07:43:04.241966168 +0300|sha256=78f0d0c132ea4b61be884501f342913fc13805900c083d82a59ddd30d067d980\nSTATUS=OK TS=2026-07-24T04:43:04Z TYPE=vm-backup BAD=0 auto-vmct=OK edge-vm-backup=OK edge-vm-offhost=OK edge-vm-restore=OK auto-timer=active edge-freshness-timer=active\nHEALTH_FILE=app-stacks-batch-backup.txt|mtime=2026-07-09 10:22:35.530260317 +0300|sha256=9218b8b34426e4ce79310c8713e165e9e1ebee79f426f29eb2efe155076b3c36\nSTATUS=OK TS=2026-07-09T07:19:28Z TYPE=app-stacks-batch-backup SOURCE=backup-sla STATUS_SOURCE=OK NOTE=legacy-health-refresh-for-prometheus-readiness\nHEALTH_FILE=blackbox-exporter-backup.txt|mtime=2026-07-09 11:04:19.961323675 +0300|sha256=2c46d63ab44ad8bd8f161a547103d8aef937643044b8b326e2b1ff609e9178d0\nSTATUS=OK TS=2026-07-09T08:04:14Z TYPE=blackbox-exporter-backup JOB=homelab-pve01-app-blackbox-exporter-20260709T080414Z SOURCE=blackbox_exporter_config SIZE=512 SHA256=31d9f2a61d820629c0e173c0e1a24cd65e24d1ef55ba0670d7fda06225cb8146 LOCAL_BACKUP_DELETED=true NOTE=service-readiness-backup-proof\nHEALTH_FILE=node-exporter-backup.txt|mtime=2026-07-09 11:14:05.435232618 +0300|sha256=9312b7986eeb48f06f3f3722fcf970624a423b4c398652d83226e59aaff4fefa\nSTATUS=OK TS=2026-07-09T08:13:40Z TYPE=node-exporter-backup JOB=homelab-pve01-app-node-exporter-20260709T081340Z SOURCE=multi_host_node_exporter_config_textfile SIZE=19018 SHA256=bb7db78e6e79c104f89c7735922e2a2e311fba242d34728e64cd60cefe69e647 LOCAL_BACKUP_DELETED=true PVE01_TEXTFILE_REPAIR=OK NOTE=service-readiness-backup-proof\nHEALTH_FILE=stirling-pdf-backup.txt|mtime=2026-07-09 21:08:55.532973745 +0300|sha256=d292f237702fbead21f1597ac86dd610f8d3a6f70b81d18d2a3e3bbd8aced551\nSTATUS=OK TS=2026-07-09T18:08:38Z TYPE=stirling-pdf-backup BAD=0 APP=stirling-pdf JOB=homelab-pve01-app-stirling-pdf-20260709T180838Z SIZE=40071 LOCAL_BACKUP_DELETED=true ENGINE=files JSON=/var/lib/homelab-backup/apps/latest-stirling-pdf.json\nHEALTH_FILE=backup-audit.json|mtime=2026-09-23 23:47:36.448063169 +0300|sha256=b193b9fc04ed47f247d4c981468387ceff69cf1415e4295ec5b4de9e08d1b6fe\n{\"cloud\":{\"A\":{\"free\":592249131673,\"free_fraction\":0.538647,\"readable\":true,\"top_dirs\":27,\"total\":1099511627776,\"used\":507262496103},\"B\":{\"free\":592249067814,\"free_fraction\":0.538647,\"readable\":true,\"top_dirs\":27,\"total\":1099511627776,\"used\":507262559962}},\"cloud_b_restore_drill\":true,\"dr_proven\":true,\"generated_at_epoch\":1790196452,\"issues\":[{\"code\":\"RPO_BREACH\",\"logical_id\":\"vm150\",\"root_cause\":\"NO_FRESH_V2_OR_SEALED_QUORUM\",\"severity\":\"HIGH\"},{\"code\":\"SOURCE_OFFHOST_MISSING\",\"logical_id\":\"-\",\"root_cause\":\"CR_BRANCH_SHA_NOT_EXACT\",\"severity\":\"MEDIUM\"}],\"schema\":\"homelab-backup-audit-v3\",\"source\":{\"local_head\":\"6401ea1e99988b9d2550de544c9e45bd7e829bdd\",\"pushurl_disabled\":true,\"remote_head\":null},\"status\":\"RED\",\"targets\":[{\"decision\":\"SKIP_FRESH\",\"generation\":\"app-filebrowser-pve01-20260923T143005Z\",\"kind\":\"APP\",\"legacy_age_seconds\":22640,\"logical_id\":\"app-filebrowser\",\"state\":\"FRESH\"},{\"decision\":\"SKIP_FRESH\",\"generation\":\"app-hbarhub-pve01-20260923T143158Z\",\"kind\":\"APP\",\"legacy_age_seconds\":22468,\"logical_id\":\"app-hbarhub\",\"state\":\"FRESH\"},{\"decision\":\"SKIP_FRESH\",\"generation\":\"app-immich-pve01-20260923T143041Z\",\"kind\":\"APP\",\"legacy_age_seconds\":22568,\"logical_id\":\"app-immich\",\"state\":\"FRESH\"},{\"decision\":\"SKIP_FRESH\",\"generation\":\"app-skladchik-reports-monitor-pve01-20260923T143025Z\",\"kind\":\"APP\",\"legacy_age_seconds\":22618,\"logical_id\":\"app-skladchik-reports-monitor\",\"state\":\"FRESH\"},{\"age_seconds\":44470,\"generation\":\"control-bootstrap-v2-pve01-20260916T120057Z\",\"kind\":\"CONTROL\",\"logical_id\":\"control-bootstrap-v2\",\"state\":\"FRESH\"},{\"generation\":\"v2-ct110-20260923T125546Z-34408ffe4182\",\"kind\":\"VM\",\"legacy_age_seconds\":1696083,\"logical_id\":\"vm110\",\"replica_count\":2,\"source\":\"V2\",\"state\":\"FRESH\",\"state_id\":\"ct110\",\"v2_age_seconds\":28097},{\"generation\":\"v2-ct111-20260923T133222Z-0d3f5fae9c35\",\"kind\":\"VM\",\"legacy_age_seconds\":null,\"logical_id\":\"vm111\",\"replica_count\":2,\"source\":\"V2\",\"state\":\"FRESH\",\"state_id\":\"ct111\",\"v2_age_seconds\":25955},{\"generation\":\"v2-ct112-20260923T200600Z-c1b47cc67dad\",\"kind\":\"VM\",\"legacy_age_seconds\":1734578,\"logical_id\":\"vm112\",\"replica_count\":2,\"source\":\"V2\",\"state\":\"FRESH\",\"state_id\":\"ct112\",\"v2_age_seconds\":2422},{\"generation\":\"v2-ct113-20260923T203800Z-bd913a8e8930\",\"kind\":\"VM\",\"legacy_age_seconds\":null,\"logical_id\":\"vm113\",\"replica_count\":2,\"source\":\"V2\",\"state\":\"FRESH\",\"state_id\":\"ct113\",\"v2_age_seconds\":497},{\"generation\":\"v2-vm130-20260923T011558Z-c9498d37d3d7\",\"kind\":\"VM\",\"legacy_age_seconds\":null,\"logical_id\":\"vm130\",\"replica_count\":2,\"source\":\"V2\",\"state\":\"FRESH\",\"state_id\":\"vm130\",\"v2_age_seconds\":69146},{\"generation\":\"vm150-pve01-20260904T053517Z\",\"kind\":\"VM\",\"legacy_age_seconds\":1696199,\"logical_id\":\"vm150\",\"replica_count\":2,\"source\":\"NONE\",\"state\":\"STALE\",\"state_id\":\"vm150\",\"v2_age_seconds\":74828},{\"generation\":\"v2-vm160-20260923T014608Z-f76d7671af5a\",\"kind\":\"VM\",\"legacy_age_seconds\":null,\"logical_id\":\"vm160\",\"replica_count\":2,\"source\":\"V2\",\"state\":\"FRESH\",\"state_id\":\"vm160\",\"v2_age_seconds\":68046},{\"generation\":\"v2-vm170-20260923T035958Z-923adacbc94b\",\"kind\":\"VM\",\"legacy_age_seconds\":640820,\"logical_id\":\"vm170\",\"replica_count\":2,\"source\":\"V2\",\"state\":\"FRESH\",\"state_id\":\"vm170\",\"v2_age_seconds\":59695},{\"generation\":\"v2-vm171-20260923T021946Z-4d6e4f578075\",\"kind\":\"VM\",\"legacy_age_seconds\":670352,\"logical_id\":\"vm171\",\"replica_count\":2,\"source\":\"V2\",\"state\":\"FRESH\",\"state_id\":\"vm171\",\"v2_age_seconds\":66249},{\"generation\":\"v2-vm190-20260923T032759Z-314d9ba2ec35\",\"kind\":\"VM\",\"legacy_age_seconds\":null,\"logical_id\":\"vm190\",\"replica_count\":2,\"source\":\"V2\",\"state\":\"FRESH\",\"state_id\":\"vm190\",\"v2_age_seconds\":62166},{\"generation\":\"v2-vm9130-20260923T071932Z-c0492db6535b\",\"kind\":\"VM\",\"legacy_age_seconds\":null,\"logical_id\":\"vm9130\",\"replica_count\":2,\"source\":\"V2\",\"state\":\"FRESH\",\"state_id\":\"vm9130\",\"v2_age_seconds\":47261}]}\nHEALTH_FILE=crypto-portfolio-ui-preapply-backup.txt|mtime=2026-07-13 10:42:57.148769166 +0300|sha256=7b4b3faf518b57a01b5bb1302377fc1fb13ad9a206f9e8211edfd8bbbc0bf395\nSTATUS=OK TS=2026-07-13T07:42:53Z TYPE=crypto-portfolio-ui-preapply-backup BAD=0 COMMAND_PHYSICAL_LINES=1 PREVIOUS_FAILURE=CORRECTED PREVIOUS_FAILURE_ROOT_CAUSE=INLINE_PYTHON_QUOTE_ERROR_IN_NONESSENTIAL_HELPER_PROOF BACKUP_SERVICE_RERUN=NO MANUAL_REHYDRATE_RERUN=NO CANONICAL_RESTORE_CHECK_RERUN=YES_READ_ONLY PRODUCTION_CHANGED=NO PRODUCTION_DATABASE_CHANGED=NO DESIRED_STATE_CHANGED=NO SERVICE_RESTARTED=NO CONTAINER_RECREATED=NO CLOUD_WRITE_RUN=NO_READ_ONLY_RESTORE_CHECK TARGET_HOST=core-apps.gram1.ru INPUT_INDEX_SHA256=8bbe88f5f420fe429d479c598e1f3f904521eb166d4707e32e579851e84c205e CANDIDATE_INDEX_SHA256=e497a36d8549654c6d451360a11a2ad714e19c23e6d384330a5436f7d32267cb BACKUP_JOB=homelab-pve01-app-crypto-portfolio-20260713T071127Z BACKUP_STATUS=success BACKUP_SCHEMA=homelab-app-backup-manifest-v2-flat BACKUP_LAYOUT=flat-v1 BACKUP_PATH_REL=hb/a/cp-260713-071127 BACKUP_DATABASE_INTEGRITY=ok BACKUP_FILE_COUNT=19 BACKUP_ARCHIVE_SIZE_BYTES=136632 BACKUP_ARCHIVE_SHA256=b29eb55820ec46c474f51e995e92e371251b6df717ac98182d807d1dbf8473f5 BACKUP_LOCAL_STAGING_DELETED=YES CANONICAL_TWO_CLOUD_RESTORE_CHECK=OK APPBACKUPCTL_RESTORE_CHECK=OK_BOTH_CLOUDS ROLLBACK_PATH=/var/lib/homelab-incidents/crypto-ui-center-alert-currency-prechange-20260713T071125Z ROLLBACK_MANIFEST_SHA256=0ef56c8f021f549777131ffe878cd5a263bdfb6f1dfb8bb080cefdd554573aca ROLLBACK_INDEX_SHA256=8bbe88f5f420fe429d479c598e1f3f904521eb166d4707e32e579851e84c205e PRODUCTION_UNCHANGED=YES PRODUCTION_HEALTH=200 VM170_FAILED_UNITS=0 PVE01_UNRELATED_FAILED_BASELINE_UNCHANGED=YES CHART_HEALTH_TIMER=ACTIVE BACKUP_TIMER=ACTIVE DESIRED_STATE_HEAD=80763a6c62b94cde4f3293f3b529980f76919d25 DESIRED_STATE_CLEAN=YES DESIRED_STATE_REMOTE_COUNT=0 READY_FOR_UI_PRODUCTION_APPLY=YES PREAPPLY_ROOT=/var/lib/homelab-change-sets/crypto-portfolio-ui-preapply-backup-canonical-close/20260713T074253Z\nHEALTH_FILE=backup-matrix-vm-health-feedback-fix.txt|mtime=2026-07-13 20:29:02.896780731 +0300|sha256=b359a93e09405221e25670600e4b45503ac30c7a8ada2f4e06f0cd9734a6e803\nSTATUS=OK TS=2026-07-13T17:29:02Z TYPE=backup-matrix-vm-health-feedback-fix BAD=0 VM_BACKUP_HEALTH_TIMER=homelab-edge-vm-offhost-freshness.timer OLD_TIMER_REENABLED=NO MATRIX_STATUS=OK JOBS_TOTAL=35 JOBS_BAD=0 JOBS_UNMAPPED=0 JOBS_DISABLED=4 READ_WRITE_OVERLAP=0 HEAVY_BACKUP_RUN=NO LIVE_SCRIPT_SHA256=a795ad5cacd48ac72e85c3a7908ad456d1e7c563ef4399e6957725146463faf2 MATRIX_CONFIG_SHA256=7613bcb25d32e61fac114da13af81e38354ec22a012a6deeaa168ee4cc454f87 MATRIX_STATE_SHA256=75f8c749e25df7842ea3be2679119b922c94c2f5ea99700b56c823cfbaedd539 ROLLBACK=/var/lib/homelab-incidents/vm-backup-health-matrix-feedback-fix-20260713T172901Z\nHEALTH_FILE=beszel-two-cloud-backup.txt|mtime=2026-07-10 22:05:25.857040628 +0300|sha256=31278a684937710c66cdabb2c4eff0c0fda078e7fef976ab8f4c49ea5778a5ca\nSTATUS=OK TS=2026-07-10T19:05:25Z TYPE=beszel-two-cloud-backup BAD=0 JOB=homelab-pve01-app-beszel-20260710T190453Z ARCHIVE_SIZE_BYTES=20068 LOCAL_STAGING_DELETED=true OFFHOST=OK RESTORE=OK\nHEALTH_FILE=cluster-admin-incident-engine-backup.txt|mtime=2026-07-11 02:30:49.625263708 +0300|sha256=5f56ac6b48e94a81f7a9ffa073525dc255b1fede76fcc04f7b629e0f51cb98ed\nSTATUS=OK TS=2026-07-10T23:30:49Z TYPE=cluster-admin-incident-engine-backup BAD=0 JOB=homelab-pve01-app-cluster-admin-incident-engine-20260710T232958Z ARCHIVE_SIZE_BYTES=157380 LOCAL_STAGING_DELETED=true OFFHOST=OK RESTORE=OK POST_FIX_CODE=included\nHEALTH_FILE=backup-unified-audit.json|mtime=2026-09-21 07:36:42.412067103 +0300|sha256=5da55c119b77511b150a6c2c7380842551293e5d3d12e807ab25291a4d9031b6\n{\n \"schema\": \"homelab-backup-unified-audit-v1\",\n \"task_id\": \"backup-unified-monitoring-v1\",\n \"change_id\": \"CR-2026-0032\",\n \"updated_at_utc\": \"2026-09-21T04:36:42Z\",\n \"audit_status\": \"OK\",\n \"backup_status\": \"RED\",\n \"risk_severity\": \"CRITICAL\",\n \"risk_reasons\": [\n \"SOURCE_DRIFT\",\n \"INFRASTRUCTURE_INVARIANT\",\n \"RPO_BREACH\",\n \"CAPACITY_CRITICAL\",\n \"RESUME_PENDING\"\n ],\n \"dr_readiness\": \"PROVEN\",\n \"dr_proof_status\": \"PROVEN\",\n \"control_freshness\": \"STALE_PROVEN\",\n \"lifecycle_status\": \"CLOSED_PROVEN\",\n \"vm_count\": 11,\n \"app_count\": 4,\n \"fresh_count\": 4,\n \"stale_count\": 10,\n \"resume_count\": 1,\n \"unverified_count\": 0,\n \"cloud_ok_count\": 2,\n \"scheduler_ok_count\": 6,\n \"source_ok_count\": 28,\n \"source_drift_count\": 1,\n \"payload_leaks\": 0,\n \"control_status\": \"STALE_PROVEN\",\n \"control_source_coverage\": \"RESTORE_EVIDENCE\",\n \"staging_status\": \"OK\",\n \"staging_bad_count\": 0,\n \"staging_warn_count\": 0,\n \"staging_node_count\": 3,\n \"escrow_status\": \"OK\",\n \"retention_status\": \"ACTIVE_OWNER_AWARE\",\n \"historical_gc_proven\": false,\n \"owner_aware_gc_proven\": true,\n \"observability_status\": \"ATTENTION\",\n \"size_anomaly_count\": 0,\n \"size_learning_count\": 0,\n \"capacity_status\": \"CRIT\",\n \"capacity_worst_used_percent\": 74,\n \"capacity_worst_days_to_80\": 3,\n \"observability_history_snapshot_count\": 336,\n \"provider_worm_proven\": false,\n \"provider_worm_skipped_by_policy\": true,\n \"clean_machine_bootstrap_proven\": true,\nHEALTH_FILE=legacy-monitoring-backups-migrated.txt|mtime=2026-07-10 23:00:03.116834953 +0300|sha256=3d8a60e0d8aeb25aed50f6eb2a92df4ef2f544b7409711f0afec378099b635c7\nSTATUS=OK TS=2026-07-10T20:00:03Z TYPE=legacy-monitoring-backups-migrated BAD=0 APPS=6 RESTORE_FAILURES=0 ACTIVE_LEGACY_STATES=0 OFFHOST=OK RESTORE=OK\nHEALTH_FILE=backup-boot-drill.json|mtime=2026-09-20 07:40:22.416980264 +0300|sha256=5cf6ed79fdec1f2901a868b05a57f17d1388189dcc2fb32e191b0ee4fd1f5d97\n{\n \"drill_executed\": false,\n \"reason\": \"BACKUP_LOCK_BUSY\",\n \"schema\": \"homelab-backup-boot-drill-health-v1\",\n \"status\": \"DEFERRED\",\n \"updated_at_utc\": \"2026-09-20T04:40:22Z\"\n}\nHEALTH_FILE=vm170-vm171-full-backup.txt|mtime=2026-08-30 21:08:08.908433073 +0300|sha256=dec2afb6c8b47c0c6be4f5a98e226ac3285524bc997cd93dd73942314e4367ee\nSTATUS=FAIL\nTS=2026-08-30T18:08:08Z\nTYPE=backup-matrix-generated\nJOB_ID=homelab_vm_full_backup\nJOB_NAME=homelab-vm-full-backup\nMATRIX_MODE=authoritative\nTIMER=homelab-vm-full-backup.timer\nSERVICE=homelab-vm-full-backup.service\nTIMER_ENABLED=not-found\nTIMER_ACTIVE=inactive\nSOURCE_KIND=health_file\nEVIDENCE_LEVEL=artifact_or_health\nSERVICE_RESULT=success\nSERVICE_EXEC_MAIN_STATUS=0\nSOURCE_HEALTH=/var/lib/homelab-backup-matrix/evidence/homelab_vm_full_backup/0-vm170-vm171-full-backup.txt\nSOURCE_STATUS=OK\nSOURCE_MTIME_UTC=2026-07-24T00:10:18Z\nAGE_SECONDS=3261470\nMAX_AGE_SECONDS=172800\nFRESHNESS=STALE\nLEGACY_SOURCE_PATH=/var/lib/homelab-health/vm170-vm171-full-backup.txt\nCANONICAL_JOB_ID=homelab_vm_full_backup\nCANONICAL_STATE=/var/lib/homelab-backup-matrix/state.json\nHEALTH_FILE=uptime-kuma-backup.txt|mtime=2026-07-09 23:39:50.594635691 +0300|sha256=62270c278bd63d2068eca9c0ee8a8036436328334a9393b357353eb8d52cc079\nSTATUS=OK TS=2026-07-09T20:39:29Z TYPE=uptime-kuma-backup BAD=0 APP=uptime-kuma JOB=homelab-pve01-app-uptime-kuma-20260709T072138Z SIZE=99851064 LOCAL_BACKUP_DELETED=true JSON=/var/lib/homelab-backup/apps/latest-uptime-kuma.json\nHEALTH_FILE=calibre-web-backup.txt|mtime=2026-07-09 21:46:20.933829215 +0300|sha256=2e841d7edd46668242924dae527480ead9c298abeb30a4c96734c382b09c90d7\nSTATUS=OK TS=2026-07-09T18:46:20Z TYPE=calibre-web-backup BAD=0 APP=calibre-web JOB=homelab-pve01-app-calibre-web-20260709T051713Z SIZE=6630 LOCAL_BACKUP_DELETED=true JSON=/var/lib/homelab-backup/apps/latest-calibre-web.json\nHEALTH_FILE=syncthing-backup.txt|mtime=2026-07-09 22:55:13.502354244 +0300|sha256=f9afb07a38c097b42a37b5a710917fafa6d31c943953d8daf6db72b7b91d9c24\nSTATUS=OK TS=2026-07-09T19:55:12Z TYPE=syncthing-backup BAD=0 APP=syncthing JOB=homelab-pve01-app-syncthing-20260709T051713Z SIZE=6732 LOCAL_BACKUP_DELETED=true JSON=/var/lib/homelab-backup/apps/latest-syncthing.json\nHEALTH_FILE=dozzle-backup.txt|mtime=2026-07-09 11:32:11.068758347 +0300|sha256=cc3f0d96c68c610cad9c1c38c70c89387617d43e1df6e0d304e4b7ae6b0464bb\nSTATUS=OK TS=2026-07-09T08:32:09Z TYPE=dozzle-backup JOB=homelab-pve01-app-dozzle-20260709T083209Z SOURCE=dozzle_stack_config SIZE=1328 SHA256=a571ee6f6012b25c16cdf15693dba29d8dd3acc3ce872b52f5c34edb81aa1ec8 LOCAL_BACKUP_DELETED=true NOTE=service-readiness-backup-proof\nHEALTH_FILE=vm-backup-policy.txt|mtime=2026-08-30 21:08:08.889182826 +0300|sha256=947eef16fbfd3815d6f823c3bc990415a33c5611342bc0d32fe959cb9e95b799\nSTATUS=FAIL\nTS=2026-08-30T18:08:08Z\nTYPE=backup-matrix-generated\nJOB_ID=homelab_vm_backup_policy\nJOB_NAME=homelab-vm-backup-policy\nMATRIX_MODE=authoritative\nTIMER=homelab-vm-backup-policy.timer\nSERVICE=homelab-vm-backup-policy.service\nTIMER_ENABLED=not-found\nTIMER_ACTIVE=inactive\nSOURCE_KIND=health_file\nEVIDENCE_LEVEL=artifact_or_health\nSERVICE_RESULT=success\nSERVICE_EXEC_MAIN_STATUS=0\nSOURCE_HEALTH=/var/lib/homelab-backup-matrix/evidence/homelab_vm_backup_policy/0-vm-backup-policy.txt\nSOURCE_STATUS=OK\nSOURCE_MTIME_UTC=2026-07-24T05:26:24Z\nAGE_SECONDS=3242504\nMAX_AGE_SECONDS=172800\nFRESHNESS=STALE\nLEGACY_SOURCE_PATH=/var/lib/homelab-health/vm-backup-policy.txt\nCANONICAL_JOB_ID=homelab_vm_backup_policy\nCANONICAL_STATE=/var/lib/homelab-backup-matrix/state.json\nHEALTH_FILE=netbird-peers.txt|mtime=2026-09-23 23:58:22.533880166 +0300|sha256=77fdd7924a7623499cc18560449d2190a7123fe7f0277c469a8a1360f440c869\nSTATUS=ERROR\nSERVICE=netbird-peers\nTIMESTAMP_UTC=20260923T205759Z\nATTEMPT_USED=2\npve01 service=active/enabled wt0_ip=[IP]/16 management=no signal=no peers=2/11 Connected\npve02 service=active/enabled wt0_ip=[IP]/16 management=no signal=no peers=2/11 Connected\npve03 service=active/enabled wt0_ip=[IP]/16 management=no signal=no peers=2/11 Connected\nedge-vm service=active/enabled wt0_ip=[IP]/16 management=yes signal=yes peers=2/11 Connected\nHEALTH_FILE=hapusya-site-backup-path-mapping.txt|mtime=2026-07-13 15:10:58.138221543 +0300|sha256=fb9db1095347b4db4dc7e37d1085cb758a5608ff8c6f3b934da00f44a905142c\nSTATUS=OK TS=2026-07-13T12:10:58Z TYPE=hapusya-site-backup-path-mapping BAD=0 MAP_SHA256=815b52dcedf7e3e92e07939bade32c7e8ed6acc0f9e064b65db6daec9f559c63 SHORT=hps LEGACY_STATE_PATH=pve01/apps/hapusya-site/homelab-pve01-app-hapusya-site-20260713T115259Z DESIRED_STATE_SNAPSHOT=/srv/homelab-desired-state/config/homelab-backup-paths.map RESTORE_CHECK=OK_BOTH_CLOUDS RETENTION=OK PRODUCTION_CHANGED=YES_COMMIT_POINT_PRESERVED\nHEALTH_FILE=crowdsec-capi-netbird-final.txt|mtime=2026-07-02 22:30:48.260143650 +0300|sha256=20b03407e972e39c3a8b7e8ca008bf4a4b6928ed3e0e9a822be886c3cd63fae0\nSTATUS=OK\nTS=20260702T193048Z\nCHECK=crowdsec-capi-netbird-final\nDETAIL=CrowdSec CAPI/LAPI stable through NetBird exit route relay [IP]; WARP absent; route smoke OK; documentation and error register updated.\nEDGE=edge-vm\nEDGE_IP=[IP]\nEDGE_NETBIRD_IP=[IP]\nPRIMARY_EXIT=relay.netbird.selfhosted\nPRIMARY_EXIT_NETBIRD_IP=[IP]\nBACKUP_EXIT=mail.netbird.selfhosted\nBACKUP_EXIT_NETBIRD_IP=[IP]\nTRACE_IP=[IP]\nTRACE_COLO=OTP\nPROOFS=771AC_REGISTER_CROWDSEC_CAPI_AFTER_EXIT_ROUTE,771AD_FINAL_CROWDSEC_CAPI_NETBIRD_CLOSURE,771AE_CROWDSEC_CAPI_SOAK_FINAL,771AF_CROWDSEC_CAPI_PROOF_INDEX\nHEALTH_FILE=cluster-admin-stage3-preapply-backup.txt|mtime=2026-07-13 06:15:56.239623410 +0300|sha256=95632c844cbdafe9be502b9164b1ec4351e55a1a0f1ae55668426c8affaa657c\nSTATUS=OK TS=2026-07-13T03:15:54Z TYPE=cluster-admin-stage3-preapply-backup BAD=0 COMMAND_PHYSICAL_LINES=1 PRODUCTION_CHANGED=EVIDENCE_ONLY LIVE_APPLICATION_CODE_CHANGED=NO DESIRED_STATE_CHANGED=NO SERVICE_RESTARTED=NO CLOUD_WRITE_RUN=NO_REUSE_VERIFIED_EXISTING_BACKUP APP_PRECHANGE_SHA256=c2748b8f96df842f491b5d2790790594222549853d0e2336cf60ecd6d4de0020 APP_CANDIDATE_SHA256=672d6a2975dd0e093788e7eb81a76e7f901ea6374ca4092a25ba0091aba1cb50 ROLLBACK_CHECKPOINT=VERIFIED REMOTE_CHECKPOINT_DIR=/var/lib/cluster-admin-stage3-preapply/20260713T030540Z CHECKPOINT_MANIFEST_SHA256=ede8c349a3456384f11460477d7cabc1a2aa0d18189ef3aa772471f2ebb26c77 CHECKPOINT_VALIDATION_WORKDIR=CORRECTED APPBACKUP=VERIFIED APPBACKUP_JOB=homelab-pve01-app-cluster-admin-incident-engine-20260713T031116Z APPBACKUP_SIZE_BYTES=1767199 APPBACKUP_RUN_LOG_EMPTY=ACCEPTED APPBACKUP_RESTORE_CHECK=OK APPBACKUP_ROUTING=VERIFIED LIVE_APP_UNCHANGED=YES PRODUCTION_RUNTIME_HEALTH=OK DESIRED_STATE_HEAD=8f888e5f2063e49da918b31584c9ea4bcf53af3a PREAPPLY_ROOT=/var/lib/homelab-stage3-preapply-appbackup/20260713T031116Z READY_FOR_STAGE3_PRODUCTION_APPLY=YES\nHEALTH_FILE=node-red-backup.txt|mtime=2026-07-09 22:46:45.732436363 +0300|sha256=cf53c240e74a9f8670ab0af52681ce9de4e45f332e70fc142925c357eaf9f00c\nSTATUS=OK TS=2026-07-09T19:46:42Z TYPE=node-red-backup BAD=0 APP=node-red JOB=homelab-pve01-app-node-red-20260709T051713Z SIZE=10521 LOCAL_BACKUP_DELETED=true JSON=/var/lib/homelab-backup/apps/latest-node-red.json\nHEALTH_FILE=proxmox-pve01-backup.txt|mtime=2026-07-09 15:29:41.985138298 +0300|sha256=2b8ca75eeec121d7e2b0c6a309d43798da5f864b57fef327f37a56b9ced3b5a8\nSTATUS=OK TS=2026-07-09T12:29:38Z TYPE=proxmox-pve01-backup JOB=homelab-pve01-app-proxmox-pve01-20260709T122938Z SOURCE=safe_pve01_config_facts SIZE=4314 LOCAL_BACKUP_DELETED=true JSON=/var/lib/homelab-backup/apps/latest-proxmox-pve01.json NOTE=service-readiness-backup-proof\nHEALTH_FILE=service-readiness-backup-health.txt|mtime=2026-07-09 09:41:40.097970342 +0300|sha256=f8074ea30fd108c0db7c85caebaaaa4a3600ea49fbea717f372bf5e6218753b4\nSTATUS=OK TS=2026-07-09T06:41:40Z TYPE=service-readiness-final BAD=0 SERVICE=backup-health HOMEPAGE=OK NPMPLUS=OK KUMA=OK ENDPOINTS=LOCAL_LAN_PUBLIC_OK CORE_GATES=OK LISTENERS=OK URL=https://backup.gram1.ru/backup.txt OVERALL=https://backup.gram1.ru/overall.txt\nHEALTH_FILE=hapusya-site-backup.txt|mtime=2026-07-13 15:10:58.137221528 +0300|sha256=5f8d4f49cf5cf6dba74912589dcd3c7fa0542171bfa31c649f6d94292fb8bb64\nSTATUS=OK TS=2026-07-13T12:10:58Z TYPE=hapusya-site-backup JOB=homelab-pve01-app-hapusya-site-20260713T115259Z ARCHIVE_SHA256=bf284292974600384c43c47fc99be6fedf0ebd68f722b402a7260f1df680d167 ARCHIVE_SIZE_BYTES=7895834 LOCAL_BACKUP_DELETED=true TWO_CLOUDS=OK ROUTE_SHA256=02d65cb0082a04cae282126cc829a002f171e863291c18ba04d788dc2d6282ca UNIT_SHA256=b3b07137b25ac0a0b4bdadadad654cf209387ade86a2ef2cdca58e8a7c0cc212 RELATIVE_TREE_SHA256=9dadf21da041bc4d6ee4a42f5b3ac7719c16bcc39052de32af895c6cd28286cb\nHEALTH_FILE=backup.txt|mtime=2026-07-24 07:04:41.721444174 +0300|sha256=88d791f9955261ebe5a65303dc418c70dca25ba23c29a92769a6785a9cd9e5bb\nSTATUS=OK\nREASON=latest backup exists, fresh, complete, and sha256 verified\nLATEST=/mnt/staging/vzdump/auto-full-backup-20260719T004537Z\nAGE_HOURS=123\nSTART=2026-07-24T07:04:28+03:00\nEND=2026-07-24T07:04:41+03:00\nRC=0\nHEALTH_FILE=npmplus-edge-backup.txt|mtime=2026-07-09 13:02:56.571425663 +0300|sha256=83e492c51913e884cad62305314e153c49bad522b9d65cddaefe337b7ed10e2d\nSTATUS=OK TS=2026-07-09T10:02:52Z TYPE=npmplus-edge-backup JOB=homelab-pve01-app-npmplus-edge-20260709T071928Z SOURCE=latest-appbackup SIZE=50316 LOCAL_BACKUP_DELETED=true JSON=/var/lib/homelab-backup/apps/latest-npmplus-edge.json NOTE=service-readiness-backup-proof\nHEALTH_FILE=p1-safe-data-backup.txt|mtime=2026-07-07 01:09:37.332826625 +0300|sha256=c1f7e49cbb188b0c1b13aed8508d267d47212490e92b6cb85a3b70ab167fd13d\nHEALTH_FILE=gitea-backup.txt|mtime=2026-08-30 21:08:08.480366809 +0300|sha256=5474fdacc557aef613bf14e7830fac12d9e616320519074a228a0a81ff6e8e68\nSTATUS=FAIL\nTS=2026-08-30T18:08:08Z\nTYPE=backup-matrix-generated\nJOB_ID=homelab_appbackup_sqlite_batch\nJOB_NAME=homelab-appbackup-sqlite-batch\nMATRIX_MODE=authoritative\nTIMER=homelab-appbackup-sqlite-batch.timer\nSERVICE=homelab-appbackup-sqlite-batch.service\nTIMER_ENABLED=not-found\nTIMER_ACTIVE=inactive\nSOURCE_KIND=systemd\nEVIDENCE_LEVEL=execution_only\nSERVICE_RESULT=success\nSERVICE_EXEC_MAIN_STATUS=0\nSOURCE_HEALTH=systemd:homelab-appbackup-sqlite-batch.service\nSOURCE_STATUS=OK\nSOURCE_MTIME_UTC=2026-07-24T01:46:28Z\nAGE_SECONDS=3255700\nMAX_AGE_SECONDS=172800\nFRESHNESS=STALE\nLEGACY_SOURCE_PATH=/var/lib/homelab-health/gitea-backup.txt\nCANONICAL_JOB_ID=homelab_appbackup_sqlite_batch\nCANONICAL_STATE=/var/lib/homelab-backup-matrix/state.json\nHEALTH_FILE=homeassistant-backup.txt|mtime=2026-07-09 22:04:04.073386425 +0300|sha256=03ce1462f536f5c3e3eb7b5224465292d1810b063946f859eb6e504eb5bd0a19\nSTATUS=OK TS=2026-07-09T19:04:00Z TYPE=homeassistant-backup BAD=0 APP=homeassistant JOB=homelab-pve01-app-homeassistant-20260709T051713Z SIZE=1855722 LOCAL_BACKUP_DELETED=true JSON=/var/lib/homelab-backup/apps/latest-homeassistant.json\nHEALTH_FILE=backup-matrix-desired-state-persistence.txt|mtime=2026-07-13 20:40:37.065354772 +0300|sha256=3d5bdf5d8ec15131c0d801287f324cbe0b6099e1fd079776ce0e3df8a5fa6144\nSTATUS=OK TS=2026-07-13T17:40:37Z TYPE=backup-matrix-desired-state-persistence BAD=0 GENERATED_HEAD=af26ec3c4829b1330d49656205ac580ce1706263 GENERATED_PARENT=25568fab24c0594418b3676dc511c8234b4e1ae5 GENERATOR_SHA256=19da7f049b4f7f7286faf292c5419c37cc1e1768c2d8fe36b257ab0cf9c18874 MATRIX_SHA256=7613bcb25d32e61fac114da13af81e38354ec22a012a6deeaa168ee4cc454f87 DESIRED_MATRIX_SHA256=7613bcb25d32e61fac114da13af81e38354ec22a012a6deeaa168ee4cc454f87 MATRIX_STATUS=OK JOBS_BAD=0 FAILED_UNITS=0 BACKUP_RUN=NO RESTORE_RUN=NO CLOUD_WRITE_RUN=NO RETENTION_RUN=NO ROLLBACK=/var/lib/homelab-incidents/backup-matrix-desired-state-persistence-20260713T174036Z\nHEALTH_FILE=alertmanager-two-cloud-backup.txt|mtime=2026-07-10 21:46:36.960914147 +0300|sha256=d90562fac18f643e765e60e54c673839c4ea57e580896241beb0ecc0ea45dea2\nSTATUS=OK TS=2026-07-10T18:46:36Z TYPE=alertmanager-two-cloud-backup BAD=0 JOB=homelab-pve01-app-alertmanager-20260710T184605Z ARCHIVE_SIZE_BYTES=1107 LOCAL_STAGING_DELETED=true OFFHOST=OK RESTORE=OK\nHEALTH_FILE=netbird-vps-trust-closure.txt|mtime=2026-07-07 18:29:49.166816011 +0300|sha256=6db8ee1ffbd2c371959917cac5ded56af3ec1829b01bd9c190e3334d6593a4a0\nSTATUS=OK TS=2026-07-07T15:29:49Z TYPE=netbird-vps-trust-closure BAD=0 WARN=0 MODE=TRUST_CLOSURE_NO_KNOWN_HOSTS_CHANGE TRUSTED_NETBIRD_IDENTITY=edge-vm TRUSTED_NETBIRD_IP=[IP] PUBLIC_VPS_IP=[IP] PUBLIC_VPS_DECISION=REJECTED_HOSTKEY_MISMATCH PUBLIC_IP_STRICT_BLOCKED=YES PUBLIC_IP_KEYSCAN_MATCH=NO EDGE_REF_OK=1 NETBIRD_PEER_OK=1 REPORT=/var/lib/homelab-netbird-vps-trust/closure-latest.md\nHEALTH_FILE=netbird-vps-trust-clean-seal.txt|mtime=2026-07-07 18:55:43.910666513 +0300|sha256=092eaa2c4f3a5f623e9440135d36e6bae7d8ae46a03d380d14a5e95657812b61\nHEALTH_FILE=searxng-backup.txt|mtime=2026-07-09 12:39:46.404327448 +0300|sha256=c1b03d9b8fee369c584a2c8f725639ed3ca77701d05dbe3b46fe5abb0f37a6b1\nSTATUS=OK TS=2026-07-09T09:39:15Z TYPE=searxng-backup JOB=homelab-pve01-app-searxng-20260709T093915Z SOURCE=searxng_stack_config_cache SIZE=738 SHA256=352c9c54437b8ceabc491452d1d7150624eda8c4bf99c28dc78fd488ec06f28c LOCAL_BACKUP_DELETED=true NOTE=service-readiness-backup-proof\nHEALTH_FILE=it-tools-backup.txt|mtime=2026-07-09 22:13:14.698027428 +0300|sha256=1481a6d2014b4caf49b7258d1922b575fadb4645a1a1878457c1f8f3eb8ca525\nSTATUS=OK TS=2026-07-09T19:13:00Z TYPE=it-tools-backup BAD=0 APP=it-tools JOB=homelab-pve01-app-it-tools-20260709T191300Z SIZE=689 LOCAL_BACKUP_DELETED=true ENGINE=files JSON=/var/lib/homelab-backup/apps/latest-it-tools.json\nHEALTH_FILE=netbird-vps-offhost.txt|mtime=2026-07-17 04:59:42.647642040 +0300|sha256=04ddca2c2a89ab344726695210540c4ca1358033f2edd22e2c97f1ab0aff4e0b\nSTATUS=OK\nSERVICE=netbird-vps-offhost\nTIMESTAMP_UTC=20260717T015941Z\nDST_HOST=[IP]\nDST=/mnt/staging/offhost/netbird-vps-backups-from-pve01\nLOCAL_LATEST=20260701T013054Z\nREMOTE_LATEST=20260701T013054Z\nREMOTE_FILES=30\nREMOTE_SIZE=466M\nHEALTH_FILE=backup-scheduler-fairness-granularity.json|mtime=2026-08-10 20:50:58.416755115 +0300|sha256=5ae14f32dd777dbf4c6dee305db5767e69b9d47edac900d014ad243d8b108fbb\n{\n \"status\": \"OK\",\n \"root_cause_count\": 0,\n \"production_changed\": true,\n \"cloud_changed\": false,\n \"backup_started\": false,\n \"applied_at_utc\": \"2026-08-10T17:50:58Z\",\n \"target_bytes\": true,\n \"global_dispatcher\": true,\n \"fairness_changed\": true,\n \"granularity_changed\": true,\n \"one_heavy_target_per_decision\": true,\n \"heavy_lock_released_between_targets\": true,\n \"no_progress_guard\": true,\n \"frozen_dependencies\": true,\n \"systemd_changed\": false,\n \"no_immediate_service_start\": true,\n \"targets\": {\n \"vm\": \"b0ca03aff1635a134b1638bf842df65e6fa26d965001b05eeb3ce68bf99811e5\",\n \"app\": \"2c52087ea5f25c2763b9126120babdaf3535afff2203bab388ba3a2372a68819\",\n \"dispatcher\": \"e83bf96c2e8290f6099e9e63033641fffd983947212ea3392d6197556a199849\"\n }\n}\nHEALTH_FILE=p2-media-backup.txt|mtime=2026-07-07 03:57:33.000124656 +0300|sha256=dccd8a9b1c9ae449783cbb8dc16ef9f2ead867b0fec20992722ec5f0aa9a4032\nHEALTH_FILE=actual-budget-backup.txt|mtime=2026-07-09 15:53:45.248005185 +0300|sha256=72d1b31e22e22ac91af66e3d8ae46b20b3b9297809ace524eece22031458fb05\nSTATUS=OK TS=2026-07-09T12:53:43Z TYPE=actual-budget-backup JOB=homelab-pve01-app-actual-budget-20260709T051713Z SOURCE=latest-appbackup SIZE=2011 LOCAL_BACKUP_DELETED=true JSON=/var/lib/homelab-backup/apps/latest-actual-budget.json NOTE=service-readiness-backup-proof\nHEALTH_FILE=backup-retention-safe-gc.json|mtime=2026-09-23 03:25:17.780212301 +0300|sha256=d5b48a3782e4181807fa6191ff5de57a1586afc151ec4cc7ba905f6764c03517\n{\n \"schema\": \"homelab-backup-retention-v1\",\n \"status\": \"OK\",\n \"task_id\": \"backup-retention-safe-gc-v1\",\n \"change_id\": \"CR-2026-0033\",\n \"updated_at_utc\": \"2026-09-23T00:25:17Z\",\n \"policy\": {\n \"vm_keep\": 2,\n \"app_keep\": 7,\n \"control_keep\": 4,\n \"min_delete_age_seconds\": 86400,\n \"gfs\": {\n \"enabled\": true,\n \"critical_vm_count\": 8,\n \"weekly_completed_buckets\": 4,\n \"monthly_completed_buckets\": 6,\n \"completed_periods_only\": true\n }\n },\n \"owner_aware_gc_proven\": true,\n \"destructive_canary_proven\": true,\n \"historical_gc_proven\": false,\n \"last_plan\": {\n \"delete_candidates\": 4,\n \"protected\": 0,\n \"kept\": 139,\n \"gfs_kept\": 28\n },\n \"last_run_deleted\": 4,\n \"legacy_namespaces_protected\": [\n \"hb\",\n \"pve01\",\n \"control-bootstrap\"\n ],\n \"provider_worm_proven\": false\n}\nHEALTH_FILE=backup-matrix-cluster-desired-state-sync.txt|mtime=2026-07-13 20:51:32.416338448 +0300|sha256=c2530de4aaee5f4c2746d4e97b7e6d9ab70c3f4059afc9bacc75fd5e1980d659\nSTATUS=OK TS=2026-07-13T17:51:31Z TYPE=backup-matrix-cluster-desired-state-sync BAD=0 PRE_SYNC_HEAD=af26ec3c4829b1330d49656205ac580ce1706263 GENERATED_SYNC_HEAD=597119391e7b265b85fd9a76644e4e33af47e469 GENERATED_SYNC_MODE=NEW_GENERATED_COMMIT GENERATED_SYNC_COMMIT_DELTA=1 MATRIX_STATUS=OK JOBS_TOTAL=35 JOBS_OK=31 JOBS_BAD=0 JOBS_UNMAPPED=0 JOBS_DISABLED=4 CLUSTER_PASSPORT=OK FINAL_READINESS=OK OVERALL=OK CLUSTER_ADMIN=OK PROBE_OK=4 PROBE_BAD=0 FAILED_UNITS=0 DESIRED_STATE_DIRTY=0 MATRIX_SHA256=7613bcb25d32e61fac114da13af81e38354ec22a012a6deeaa168ee4cc454f87 MATRIX_STATE_SHA256=14a31112aac5d577ae01df567921b093b2fedcf7189c4cc8f009216ee0749380 PRESEAL_SHA256=7ea1f710cd311bd51dba05a313b4c540f57d97407f008881e502fc213b4b3c99 BACKUP_RUN=NO RESTORE_RUN=NO CLOUD_WRITE_RUN=NO RETENTION_RUN=NO EVIDENCE_ROOT=/var/lib/homelab-change-sets/backup-matrix-cluster-final-seal/20260713T175131Z\nHEALTH_FILE=control-backup.json|mtime=2026-09-23 11:26:22.485179218 +0300|sha256=0755695105c316d7ee6f3926a1ded277c1c878f92e0691e62eed98f90836dc0b\n{\n \"schema\": \"homelab-control-backup-health-v1\",\n \"status\": \"OK\",\n \"generation\": \"control-bootstrap-v2-pve01-20260916T120057Z\",\n \"state_file\": \"/var/lib/homelab-backup/cloud-quorum/control-bootstrap-v2/control-bootstrap-v2-pve01-20260916T120057Z.json\",\n \"restore_evidence\": \"/var/lib/homelab-backup/control-restore-validation/control-bootstrap/control-bootstrap-v2-pve01-20260916T120057Z.json\",\n \"archive_sha256\": \"6fa394732cc1cafbf311a022b5fbdb6cf41a6562d9b3503666525c9392503e29\",\n \"cloud_quorum\": 2,\n \"control_freshness\": \"FRESH\",\n \"full_site_dr_proven\": false,\n \"updated_at_utc\": \"2026-09-23T08:26:22Z\"\n}\nHEALTH_FILE=homelab-backup-baseline.txt|mtime=2026-06-17 23:02:13.952559036 +0300|sha256=4959757d29bb503315c60468dc0eb080b43cefde23d48e8ab192f6bd93b564d3\nSTATUS=OK\nSERVICE=homelab-backup-baseline\nTIMESTAMP_UTC=20260617T200213Z\nENDPOINT_TOTAL=17\nENDPOINT_OK=17\nENDPOINT_BAD=0\nKUMA_TOTAL=18\nKUMA_UP=18\nKUMA_BAD=0\nENDPOINT_SUMMARY=disk-space.txt:OK;immich-media-backup.txt:OK;immich-media-offhost.txt:OK;memos-backup.txt:OK;memos-offhost.txt:OK;paperless-backup.txt:OK;paperless-offhost.txt:OK;netbird-vps-backup.txt:OK;netbird-vps-offhost.txt:OK;netbird-vps-restore.txt:OK;netbird-peers.txt:OK;filebrowser-backup.txt:OK;filebrowser-offhost.txt:OK;filebrowser-restore.txt:OK;app-stacks-batch-backup.txt:OK;app-stacks-batch-offhost.txt:OK;app-stacks-batch-restore.txt:OK;\nKUMA_MONITORS=1|Homelab Backup Health|active=1|user_id=1|latest_status=1|url=http://[IP]:9101/backup.txt;24|Immich Media Backup|active=1|user_id=1|latest_status=1|url=http://[IP]:9101/immich-media-backup.txt;25|Immich Media Offhost Backup|active=1|user_id=1|latest_status=1|url=http://[IP]:9101/immich-media-offhost.txt;26|Disk Space Health|active=1|user_id=1|latest_status=1|url=http://[IP]:9101/disk-space.txt;27|Memos Backup|active=1|user_id=1|latest_status=1|url=http://[IP]:9101/memos-backup.txt;28|Memos Offhost Backup|active=1|user_id=1|latest_status=1|url=http://[IP]:9101/memos-offhost.txt;29|Paperless Backup|active=1|user_id=1|latest_status=1|url=http://[IP]:9101/paperless-backup.txt;30|Paperless Offhost Backup|active=1|user_id=1|latest_status=1|url=http://[IP]:9101/paperless-offhost.txt;31|NetBird VPS Backup|active=1|user_id=1|latest_status=1|url=http://[IP]:9101/netbird-vps-backup.txt;32|NetBird VPS Offhost Backup|active=1|user_id=1|latest_status=1|url=http://[IP]:9101/netbird-vps-offhost.txt;33|NetBird VPS Restore Validation|active=1|user_id=1|latest_status=1|url=http://[IP]:9101/netbird-vps-restore.txt;34|NetBird Homelab Peers|active=1|user_id=1|latest_status=1|url=http://[IP]:9101/netbird-peers.txt;35|File Browser Backup|active=1|user_id=1|latest_status=1|url=http://[IP]:9101/filebrowser-backup.txt;36|File Browser Offhost Backup|active=1|user_id=1|latest_status=1|url=http://[IP]:9101/filebrowser-offhost.txt;37|File Browser Restore Validation|active=1|user_id=1|latest_status=1|url=http://[IP]:9101/filebrowser-restore.txt;38|App Stacks Batch Backup|active=1|user_id=1|latest_status=1|url=http://[IP]:9101/app-stacks-batch-backup.txt;39|App Stacks Batch Offhost|active=1|user_id=1|latest_status=1|url=http://[IP]:9101/app-stacks-batch-offhost.txt;40|App Stacks Batch Restore|active=1|user_id=1|latest_status=1|url=http://[IP]:9101/app-stacks-batch-restore.txt;\nNOTES=none\nHEALTH_FILE=memos-backup.txt|mtime=2026-08-30 21:08:09.015289529 +0300|sha256=01edf9ba034eb56e7a2ae8abd4ff6ee677fd19c320b4aedefe85d574fc46ee35\nSTATUS=FAIL\nTS=2026-08-30T18:08:08Z\nTYPE=backup-matrix-generated\nJOB_ID=memos_backup\nJOB_NAME=memos-backup\nMATRIX_MODE=authoritative\nTIMER=memos-backup.timer\nSERVICE=memos-backup.service\nTIMER_ENABLED=not-found\nTIMER_ACTIVE=inactive\nSOURCE_KIND=health_file\nEVIDENCE_LEVEL=artifact_or_health\nSERVICE_RESULT=success\nSERVICE_EXEC_MAIN_STATUS=0\nSOURCE_HEALTH=/var/lib/homelab-backup/apps/latest-memos.json\nSOURCE_STATUS=OK\nSOURCE_MTIME_UTC=2026-08-06T21:26:13Z\nAGE_SECONDS=2061715\nMAX_AGE_SECONDS=259200\nFRESHNESS=STALE\nLEGACY_SOURCE_PATH=/var/lib/homelab-health/memos-backup.txt\nCANONICAL_JOB_ID=memos_backup\nCANONICAL_STATE=/var/lib/homelab-backup-matrix/state.json\nHEALTH_FILE=netbird-vps-backup.txt|mtime=2026-07-01 04:31:05.433646905 +0300|sha256=372804695d6dba94e29e254f2175701e976692e7b15ee83fdcb6f10e05ca624e\nSTATUS=OK\nSERVICE=netbird-vps-backup\nTIMESTAMP_UTC=20260701T013054Z\nVPS_IP=[IP]\nSNAPSHOT=/mnt/staging/netbird-vps-backups/snapshots/20260701T013054Z\nLATEST_SYMLINK=/mnt/staging/netbird-vps-backups/snapshots/latest\nFILES=2\nSIZE=32M\nSHA256=0226dd820cb9ede0ea9e8ea4af6cef656ffa54cfb819048c222a88f4c14d8db9\nHEALTH_FILE=backup-matrix-cluster-preseal-convergence.txt|mtime=2026-07-13 20:48:14.631325305 +0300|sha256=7ea1f710cd311bd51dba05a313b4c540f57d97407f008881e502fc213b4b3c99\nSTATUS=OK TS=2026-07-13T17:48:14Z TYPE=backup-matrix-cluster-preseal-convergence BAD=0 MATRIX_STATUS=OK JOBS_TOTAL=35 JOBS_BAD=0 JOBS_UNMAPPED=0 JOBS_DISABLED=4 CLUSTER_PASSPORT=OK FINAL_READINESS=OK OVERALL=OK CLUSTER_ADMIN=OK PROBE_OK=4 PROBE_BAD=0 WEBPANEL=OK FAILED_UNITS=0 DESIRED_STATE_HEAD=af26ec3c4829b1330d49656205ac580ce1706263 DESIRED_STATE_DIRTY=0 MATRIX_SHA256=7613bcb25d32e61fac114da13af81e38354ec22a012a6deeaa168ee4cc454f87 MATRIX_STATE_SHA256=3791fa78cdfb56cd7027f89350dff245abcffc4f935adfc336ceeea00854ac89 CLUSTER_PASSPORT_SHA256=1b8b924ae0e5ce6872fe5671318094ce494806c2db7efe28299b5ff49adc0d0d FINAL_READINESS_SHA256=ac5159fa8c497afe021b5cb232f08cfec0e1a4d516d8aae966ba03f7034f1412 OVERALL_SHA256=6ae529bfdf5811f686d3b53978f453ce4f8757c74fc1be105de219b13a61ba46 BACKUP_RUN=NO RESTORE_RUN=NO CLOUD_WRITE_RUN=NO RETENTION_RUN=NO EVIDENCE_ROOT=/var/lib/homelab-change-sets/backup-matrix-cluster-preseal-resume/20260713T174748Z\nHEALTH_FILE=cluster-admin-stage4f-preapply-backup.txt|mtime=2026-07-14 10:54:50.681548259 +0300|sha256=cc242d1b6e2ef6b6d25f283c7d967bd4985222783ce65050d75e6e61230067ae\nSTATUS=OK TS=2026-07-14T07:54:50Z TYPE=cluster-admin-stage4f-preapply-backup-final-seal BAD=0 STAGE4F_CLOSED=YES NEXT_STAGE=4G_PRODUCTION_APPLY_READ_ONLY_ONLY PRODUCTION_CHANGED=NO DATABASE_CHANGED=NO APPLICATION_CHANGED=NO BACKUP_JOB=homelab-pve01-app-cluster-admin-incident-engine-20260714T064327Z ARCHIVE_SIZE_BYTES=2651545 DATABASE_DUMP_SIZE_BYTES=2814735 CURRENT_JOB_VERIFY_MARKERS=2 RESTORE_LIST_CHECK=OK PRODUCTION_STATE=0|0|OK DESIRED_STATE_HEAD=69be75caeecaa242567d2d06c735ac5db3a9c048 DESIRED_STATE_CLEAN=YES REPORT=/var/lib/homelab-cluster-admin-stage4/stage4f-preapply-backup-final-seal-v1/report.txt REPORT_SHA256=9ee244085b9d191370371560690ea87e7bbfe2f14226a06a4e8f6720b3a7439b\nHEALTH_FILE=jellyfin-backup.txt|mtime=2026-07-09 21:16:17.662793576 +0300|sha256=23afe6670bf9250dada540261513ffff2c36ee33963347e5673699d4a10f77a2\nSTATUS=OK TS=2026-07-09T18:16:16Z TYPE=jellyfin-backup BAD=0 APP=jellyfin JOB=homelab-pve01-app-jellyfin-20260709T051713Z SIZE=25920 LOCAL_BACKUP_DELETED=true JSON=/var/lib/homelab-backup/apps/latest-jellyfin.json\nHEALTH_FILE=vikunja-backup.txt|mtime=2026-07-09 20:51:20.850795887 +0300|sha256=48bc526e9c24220ad6c15554b85b819d8b6ba35d854ab87da2324b1534e194c6\nSTATUS=OK TS=2026-07-09T17:51:14Z TYPE=vikunja-backup BAD=0 APP=vikunja JOB=homelab-pve01-app-vikunja-20260709T051713Z SIZE=34831 LOCAL_BACKUP_DELETED=true JSON=/var/lib/homelab-backup/apps/latest-vikunja.json\nHEALTH_FILE=hapusya-site-v2-preapply-backup.txt|mtime=2026-07-13 14:34:59.208365351 +0300|sha256=506879fe26ddf3c392c4d485d1db145cde846336cd626096555e2021ab2705af\nSTATUS=OK TS=2026-07-13T11:34:59Z TYPE=hapusya-site-v2-preapply-backup BAD=0 NPMPLUS_JOB=homelab-pve01-app-npmplus-edge-20260713T113435Z TWO_CLOUDS=OK RESTORE_CHECK=OK ROLLBACK=/var/lib/homelab-incidents/hapusya-site-v2-preapply-20260713T113458Z ROLLBACK_MANIFEST_SHA256=144506c766fe9fd3152e4edf94a8867895b3dc54f29faf1e329773922b8c5640 CANDIDATE_TREE_SHA256=0eb098f91ea706c884d36a8ce806968e82e715ded66bb96f61c2ed8106faca93 CANDIDATE_BROWSER=OK BROWSER_RUNS=5 VIDEO_DESKTOP=OK VIDEO_MOBILE=OK STATIC_FALLBACK=OK PRODUCTION_CHANGED=NO\nHEALTH_FILE=ntfy-backup.txt|mtime=2026-07-09 23:39:41.327493875 +0300|sha256=e28d674bd0ba4c088c6c9f523da493e3674750837ab417223f7405ea61d8b020\nSTATUS=OK TS=2026-07-09T20:39:29Z TYPE=ntfy-backup BAD=0 APP=ntfy JOB=homelab-pve01-app-ntfy-20260709T051713Z SIZE=13516 LOCAL_BACKUP_DELETED=true JSON=/var/lib/homelab-backup/apps/latest-ntfy.json\nHEALTH_FILE=immich-media-backup.txt|mtime=2026-08-30 21:08:08.994511799 +0300|sha256=f94d804693a2ca18e51c66ade993a95c9b8ae183730e31013e3886010b03ff63\nSTATUS=FAIL\nTS=2026-08-30T18:08:08Z\nTYPE=backup-matrix-generated\nJOB_ID=immich_media_backup\nJOB_NAME=immich-media-offhost-sync\nMATRIX_MODE=authoritative\nTIMER=immich-media-offhost-sync.timer\nSERVICE=immich-media-offhost-sync.service\nTIMER_ENABLED=not-found\nTIMER_ACTIVE=inactive\nSOURCE_KIND=health_file\nEVIDENCE_LEVEL=artifact_or_health\nSERVICE_RESULT=success\nSERVICE_EXEC_MAIN_STATUS=0\nSOURCE_HEALTH=/var/lib/homelab-health/immich-media-offhost.txt\nSOURCE_STATUS=OK\nSOURCE_MTIME_UTC=2026-08-08T05:52:15Z\nAGE_SECONDS=1944953\nMAX_AGE_SECONDS=259200\nFRESHNESS=STALE\nLEGACY_SOURCE_PATH=/var/lib/homelab-health/immich-media-backup.txt\nCANONICAL_JOB_ID=immich_media_backup\nCANONICAL_STATE=/var/lib/homelab-backup-matrix/state.json\nHEALTH_FILE=paperless-backup.txt|mtime=2026-07-18 01:52:29.893166209 +0300|sha256=8ae7e31227e48c86412eb9e49fa91ce3cbb97454ea3414e25d10ef5432e6cd46\nSTATUS=OK\nTS=2026-07-17T22:52:29Z\nTYPE=backup-matrix-generated\nJOB_ID=paperless_backup\nJOB_NAME=paperless-backup\nMATRIX_MODE=authoritative\nTIMER=paperless-backup.timer\nSERVICE=paperless-backup.service\nTIMER_ENABLED=enabled\nTIMER_ACTIVE=active\nSOURCE_KIND=health_file\nEVIDENCE_LEVEL=artifact_or_health\nSERVICE_RESULT=success\nSERVICE_EXEC_MAIN_STATUS=0\nSOURCE_HEALTH=/var/lib/homelab-backup/apps/latest-paperless.json\nSOURCE_STATUS=OK\nSOURCE_MTIME_UTC=2026-07-17T02:30:25Z\nAGE_SECONDS=73324\nMAX_AGE_SECONDS=259200\nFRESHNESS=FRESH\nLEGACY_SOURCE_PATH=/var/lib/homelab-health/paperless-backup.txt\nCANONICAL_JOB_ID=paperless_backup\nCANONICAL_STATE=/var/lib/homelab-backup-matrix/state.json\nHEALTH_FILE=cluster-admin-stage1-preapply-backup.txt|mtime=2026-07-13 03:11:06.700511934 +0300|sha256=a70d7bba3c2b50deaa117e56862659b857e72682ec829ed7724b03dc92111105\nSTATUS=OK TS=2026-07-13T00:10:57Z TYPE=cluster-admin-stage1-preapply-backup BAD=0 PRODUCTION_CHANGED=BACKUP_ARTIFACTS_ONLY PRODUCTION_DATABASE_CHANGED=NO SERVICE_RESTARTED=NO CLOUD_WRITE_RUN=NO ROLLBACK_POINT=VERIFIED DIRECT_DB_SNAPSHOT=VERIFIED DIRECT_DB_DUMP_SIZE_BYTES=1710825 DIRECT_DB_DUMP_SHA256=c0f13cec235ab3242eb8da4cad12962df1a2e37b25792a8ce9944c1d7de4312a APPBACKUP_JOB=homelab-pve01-app-cluster-admin-incident-engine-20260713T000543Z APPBACKUP_STATUS=success APPBACKUP_RESTORE_CHECK=OK MIGRATION_DRY_RUN=OK IDEMPOTENCE_TEST=OK ROLLBACK_TEST=OK COLLECTOR_COMPATIBILITY_TRIGGER=OK PRODUCTION_SCHEMA_UNCHANGED=YES READY_FOR_PRODUCTION_APPLY=YES DESIRED_STATE_HEAD=1379b4fb4e809e56e9bc16f0e0b19b0a509192d5 PREAPPLY_ROOT=/var/lib/homelab-stage1-preapply/20260713T000041Z REMOTE_SNAPSHOT_DIR=/var/lib/cluster-admin-preapply/20260713T000041Z EVIDENCE_MANIFEST=/var/lib/homelab-stage1-preapply/20260713T000041Z/SHA256SUMS\nHEALTH_FILE=cadvisor-backup.txt|mtime=2026-07-09 11:25:57.266067724 +0300|sha256=cb0bcacf449ab29e8954014fb689d1022527f0b67aae0d4f0c808358cae55e07\nSTATUS=OK TS=2026-07-09T08:25:56Z TYPE=cadvisor-backup JOB=homelab-pve01-app-cadvisor-20260709T082556Z SOURCE=cadvisor_stack_proxy SIZE=1384 SHA256=33e419912291daab19f8039dfe8971f277be0fe577eb3d0e538263bdf918eac8 LOCAL_BACKUP_DELETED=true NOTE=service-readiness-backup-proof\nHEALTH_FILE=dockge-stacks-backup.txt|mtime=2026-07-09 13:12:46.609379653 +0300|sha256=f20ebfd8e230858d31286670ccb6a4699d4251209765fb37e3ff4309435113f5\nSTATUS=OK TS=2026-07-09T10:12:44Z TYPE=dockge-stacks-backup JOB=homelab-pve01-app-dockge-stacks-20260709T072218Z SOURCE=latest-appbackup SIZE=134913 LOCAL_BACKUP_DELETED=true JSON=/var/lib/homelab-backup/apps/latest-dockge-stacks.json NOTE=service-readiness-backup-proof\nHEALTH_FILE=dockge-backup.txt|mtime=2026-07-09 13:12:46.609379653 +0300|sha256=3f5198e9d0eb7852e4ee563109c801be03be268b2b10afeca68fbc39be4d35f7\nSTATUS=OK TS=2026-07-09T10:12:44Z TYPE=dockge-backup JOB=homelab-pve01-app-dockge-stacks-20260709T072218Z SOURCE=appbackup-dockge-stacks SIZE=134913 LOCAL_BACKUP_DELETED=true JSON=/var/lib/homelab-backup/apps/latest-dockge.json NOTE=service-readiness-backup-proof\nHEALTH_FILE=backup-lock-fd-hardening.json|mtime=2026-08-10 17:49:25.372646496 +0300|sha256=d64d18259e58525a9549f6d2b40b331ea6cc7b7057fd17050aebc585d73731a5\n{\n \"status\": \"OK\",\n \"root_cause_count\": 0,\n \"production_changed\": true,\n \"cloud_changed\": false,\n \"backup_started\": false,\n \"applied_at_utc\": \"2026-08-10T14:49:25Z\",\n \"target_bytes\": true,\n \"fd_child_boundary\": true,\n \"parent_lock_ownership_preserved\": true,\n \"systemd_changed\": false,\n \"no_immediate_service_start\": true,\n \"targets\": {\n \"vm\": \"55c2c30d93590ce0367667c2de4cf8e6436bd10a2a35fb7b775f918ba9b52ffb\",\n \"app\": \"adcd592c619c96c99be19777eff98bbf0226e878a9635c718e9113b662960a5a\"\n }\n}\nHEALTH_FILE=backup-sla.txt|mtime=2026-08-30 21:08:08.601604759 +0300|sha256=4762bb7cfb98ab1311adbd1e80287f7e18f79983656a888a056ed730ecdf0372\nSTATUS=FAIL\nTS=2026-08-30T18:08:08Z\nTYPE=backup-matrix-generated\nJOB_ID=homelab_backup_sla_health\nJOB_NAME=homelab-backup-sla-health\nMATRIX_MODE=authoritative\nTIMER=homelab-backup-sla-health.timer\nSERVICE=homelab-backup-sla-health.service\nTIMER_ENABLED=not-found\nTIMER_ACTIVE=inactive\nSOURCE_KIND=health_file\nEVIDENCE_LEVEL=artifact_or_health\nSERVICE_RESULT=success\nSERVICE_EXEC_MAIN_STATUS=0\nSOURCE_HEALTH=/var/lib/homelab-backup-matrix/evidence/homelab_backup_sla_health/0-backup-sla.txt\nSOURCE_STATUS=WARN\nSOURCE_MTIME_UTC=2026-08-06T21:34:03Z\nAGE_SECONDS=2061245\nMAX_AGE_SECONDS=172800\nFRESHNESS=STALE\nLEGACY_SOURCE_PATH=/var/lib/homelab-health/backup-sla.txt\nCANONICAL_JOB_ID=homelab_backup_sla_health\nCANONICAL_STATE=/var/lib/homelab-backup-matrix/state.json\nHEALTH_FILE=deep-service-backup-audit.txt|mtime=2026-07-10 00:12:45.326483756 +0300|sha256=16dfd49e8f0618a836c473874b7a70fa896771dff5e2613e510e6d0a02eca7b4\nSTATUS=OK TS=2026-07-09T21:12:44Z TYPE=deep-service-backup-audit BAD=0 SERVICE_READINESS_OK_COUNT=47 NON_OK=0 APPBACKUP_FLAG_GAPS=0 CONTAINER_GAPS=0 PROM_FIRING=0 BACKUP_SLA=OK ALERTING=OK EXTERNAL_LINKS_CLASSIFIED=OK SERVICES_REMAINING_TOTAL=0 BACKUP_GAPS_TOTAL=0 COMMANDS_REMAINING_TOTAL=0\nHEALTH_FILES_END=1\nDEPENDENCY_FILE_INDEX_BEGIN=1\nDEPENDENCY_FILE_COUNT=34\nDEPENDENCY_FILE|path=/etc/homelab-backup-v2/targets.json|mtime=2026-09-22 12:19:09.941039348 +0300|sha256=decea6b876c8ede15c91cad69c8e15e46021e5e96d25d415139bf2c524e23f7b\n2: \"created_by\": \"SUPPORT-260914-INFRA-DR-BACKUPV2-SHADOW-PERSISTENT-PRODUCERS-STAGE-GREEN1-389R1\",\n169: \"logical_id\": \"md-relay\",\n193: \"logical_id\": \"us-netbird\",\nDEPENDENCY_FILE|path=/etc/systemd/system/netbird-vps-backup.service.d/10-superseded-noop.conf|mtime=2026-07-08 08:25:22.050610124 +0300|sha256=4cf3f2825b770b32447624a611bce605d97987bd2c6e8862c84d35b166f9102a\n3:ExecStart=/usr/local/sbin/homelab-superseded-unit-ok netbird-vps-backup.service superseded_by_current_appbackup_or_trust_proof\nDEPENDENCY_FILE|path=/root/771ad_final_crowdsec_capi_netbird_closure.sh|mtime=2026-07-02 22:10:57.781648619 +0300|sha256=c99d08385f5d7c0a266c1c8002b3c7b054e1bb561eb48b1f8a227f37fe430ab6\n8:PROOF=\"/root/evidence/771AD_FINAL_CROWDSEC_CAPI_NETBIRD_CLOSURE_${TS}_PROOF.txt\"\n24: echo \"STEP=771AD_FINAL_CROWDSEC_CAPI_NETBIRD_CLOSURE\"\n47: echo NETBIRD_EXIT_ROUTE_VERIFY\n48: netbird status 2>&1 | redact || true\n59: curl -4 -sk --http1.1 --connect-timeout 12 --max-time 45 -o /tmp/771ad_capi.body -w \\\"CAPI_HTTP=%{http_code}\\\\n\\\" https://api.crowdsec.net/v3/watchers/login || true\n61: echo CROWDSEC_STATUS\n62: docker ps --format \\\"{{.Names}} {{.Status}} {{.Ports}}\\\" | grep -E \\\"^(crowdsec|socket-proxy|npmplus)\\\" || true\n63: docker exec crowdsec cscli lapi status 2>&1 | redact || true\n64: docker exec crowdsec cscli capi status 2>&1 | redact || true\n67: cd /opt/stacks/crowdsec\n78: docker logs --since 10m crowdsec 2>&1 | grep -Ei \\\"fatal|TLS handshake timeout|unable to run local API|failed to get docker info|lookup socket-proxy|server misbehaving|mapping value is not allowed|missing credentials field|online_api_credentials\\\" | redact || echo RECENT_FATALS_NONE\n80: echo HEALTH_FILE\n81: cat /var/lib/homelab-health/crowdsec-capi.txt 2>/dev/null | redact || true\n97: if ! grep -q 'CROWDSEC_CAPI_NETBIRD_EXIT_ROUTE_20260702' \"$REF\"; then\n100:## CROWDSEC_CAPI_NETBIRD_EXIT_ROUTE_20260702\n101:- edge-vm CrowdSec CAPI is registered and enabled through NetBird exit route.\n102:- Client peer: edge-vm.netbird.selfhosted / [IP].\n103:- Primary exit routing peer: relay.netbird.selfhosted / [IP] / Moldova.\n104:- Backup egress peer present: mail.netbird.selfhosted / [IP] / USA.\n105:- NetBird exit route proof on edge-vm: wg allowed-ips includes [IP]/0 via relay; ip route get [IP] uses wt0 table 7120.\n107:- CrowdSec proof: cscli lapi status OK, cscli capi status OK, CAPI sharing/blocklist pull enabled, no DISABLE_ONLINE_API or -no-capi flags in active compose.\n109:- Unauthenticated https://api.crowdsec.net/v3/watchers/login returning HTTP 403 is expected network reachability proof.\n118: if ! grep -q 'LESSON_20260702_CROWDSEC_NETBIRD_EXIT_ROUTE' \"$ERR\"; then\n121:## LESSON_20260702_CROWDSEC_NETBIRD_EXIT_ROUTE\n122:- Do not treat e3qxxx.netbird.selfhosted / [IP] and e3qxxx-183-106.netbird.selfhosted / [IP] as VPS egress nodes; they are user mobile peers.\n123:- Correct egress candidates are relay.pvepro.ru / relay.netbird.selfhosted / [IP] for Moldova and mail.pvepro.ru / mail.netbird.selfhosted / [IP] for USA.\n124:- Access Policy edge-vm -> relay/mail only proves peer reachability. It is not an internet exit route.\n125:- Required NetBird fix for CrowdSec CAPI was Network Routing exit route [IP]/0 distributed to edge-vm group, routing peer relay, masquerade/NAT enabled.\n126:- Before attempting CrowdSec CAPI, verify on edge-vm: ip route get [IP] uses wt0, wg allowed-ips contains [IP]/0, and Cloudflare trace no longer shows home IP [IP].\n134: echo \"STATUS=OK_771AD_FINAL_CROWDSEC_CAPI_NETBIRD_CLOSURE_DONE\"\nDEPENDENCY_FILE|path=/root/771ag_publish_final_health_index.sh|mtime=2026-07-02 22:30:48.251143514 +0300|sha256=8b7c814c3893ebb2068d90c29ec01460895aba7cba44d78ba8d7d2a8936c1b7a\n8:PROOF=\"/root/evidence/771AG_PUBLISH_FINAL_HEALTH_INDEX_${TS}_PROOF.txt\"\n10:LOCAL_HEALTH=\"/var/lib/homelab-health/crowdsec-capi-netbird-final.txt\"\n23: echo \"STEP=771AG_PUBLISH_FINAL_HEALTH_INDEX\"\n26: echo \"CONFIG_CHANGE=NO_SERVICE_CHANGE_HEALTH_INDEX_ONLY\"\n31: grep -n 'CROWDSEC_CAPI_NETBIRD_EXIT_ROUTE_20260702' /etc/pve/31_HOMELAB_REFERENCE.md || true\n32: grep -n 'LESSON_20260702_CROWDSEC_NETBIRD_EXIT_ROUTE' /etc/pve/HOMELAB_ASSISTANT_ERROR_REGISTER.md || true\n34: echo \"WRITE_LOCAL_HEALTH_SUMMARY\"\n35: mkdir -p /var/lib/homelab-health\n36: cat >\"$LOCAL_HEALTH\" </dev/null | redact || true\n77: echo EDGE_CROWDSEC\n78: docker exec crowdsec cscli lapi status 2>&1 | redact || true\n79: docker exec crowdsec cscli capi status 2>&1 | redact || true\n82: docker inspect crowdsec --format \\\"RestartCount={{.RestartCount}}\\\" 2>/dev/null || true\n83: cd /opt/stacks/crowdsec\n93: for cmd in /usr/local/sbin/homelab-health-metrics /usr/local/sbin/homelab-evidence-catalog.sh; do\n111: echo \"LOCAL_HEALTH_FINAL\"\n112: cat \"$LOCAL_HEALTH\"\n115: echo \"STATUS=OK_771AG_PUBLISH_FINAL_HEALTH_INDEX_DONE\"\nDEPENDENCY_FILE|path=/root/771ak_final_done_marker.sh|mtime=2026-07-02 22:34:56.049721292 +0300|sha256=64f3b7e547623a9908eab9f09a1edcaeac8795af30db85e975dfa2e95cbe0d2a\n15: echo \"RESULT=CROWDSEC_CAPI_CLOSED\"\n18: echo \"EXIT_ROUTE=relay.netbird.selfhosted/[IP]\"\n23: echo \"REFERENCE_MARKER_LINE=$(grep -n 'CROWDSEC_CAPI_NETBIRD_EXIT_ROUTE_20260702' /etc/pve/31_HOMELAB_REFERENCE.md | cut -d: -f1 | head -1)\"\n24: echo \"ERROR_REGISTER_MARKER_LINE=$(grep -n 'LESSON_20260702_CROWDSEC_NETBIRD_EXIT_ROUTE' /etc/pve/HOMELAB_ASSISTANT_ERROR_REGISTER.md | cut -d: -f1 | head -1)\"\n25: echo \"FINAL_HEALTH=$(grep -m1 '^STATUS=' /var/lib/homelab-health/crowdsec-capi-netbird-final.txt 2>/dev/null || echo MISSING)\"\n26: echo \"SOAK_HEALTH_ON_EDGE=$(ssh debian@[IP] 'sudo grep -m1 \"^STATUS=\" /var/lib/homelab-health/crowdsec-capi.txt 2>/dev/null || echo MISSING')\"\n28: ls -1 /root/evidence/771AC_REGISTER_CROWDSEC_CAPI_AFTER_EXIT_ROUTE_20260702T190428Z_PROOF.txt \\\n29: /root/evidence/771AD_FINAL_CROWDSEC_CAPI_NETBIRD_CLOSURE_20260702T191057Z_PROOF.txt \\\n30: /root/evidence/771AE_CROWDSEC_CAPI_SOAK_FINAL_20260702T191745Z_PROOF.txt \\\n31: /root/evidence/771AF_CROWDSEC_CAPI_PROOF_INDEX_20260702T192904Z.txt \\\n32: /root/evidence/771AG_PUBLISH_FINAL_HEALTH_INDEX_20260702T193048Z_PROOF.txt \\\nDEPENDENCY_FILE|path=/root/771l_crowdsec_capi_netbird_vps_egress_remote_20260702T151045Z.sh|mtime=2026-07-02 18:10:45.300196691 +0300|sha256=6a6df3cc085363c2bbbb92b8c3083123fb0e583e14e242c583a321bdb5612002\n5:STACK=/opt/stacks/crowdsec\n7:BACKUP_DIR=\"$STACK/manual-backups/771l-netbird-vps-egress-$TS\"\n8:HEALTH_DIR=/var/lib/homelab-health\n9:HEALTH_FILE=\"$HEALTH_DIR/crowdsec-capi.txt\"\n10:EGRESS_DIR=/etc/homelab-crowdsec-capi-netbird-egress\n11:EGRESS_SERVICE=homelab-crowdsec-capi-netbird-egress.service\n14:mkdir -p \"$BACKUP_DIR\" \"$HEALTH_DIR\" \"$EGRESS_DIR/profiles\"\n20:write_health() {\n26: echo \"CHECK=crowdsec-capi\"\n29: } > \"$HEALTH_FILE\"\n33: docker network inspect crowdsec_default --format '{{(index .IPAM.Config 0).Gateway}}' 2>/dev/null || echo \"[IP]\"\n41: crowdsec:\n66: docker compose up -d --force-recreate crowdsec\n74: state=\"$(docker inspect crowdsec --format 'Running={{.State.Running}} Restarting={{.State.Restarting}} RestartCount={{.RestartCount}} StartedAt={{.State.StartedAt}} ExitCode={{.State.ExitCode}}' 2>/dev/null || true)\"\n75: health=\"$(curl -sk --connect-timeout 3 --max-time 6 -o /tmp/771l_health.html -w '%{http_code}' http://[IP]:8088/health || true)\"\n78: fatal=\"$(docker logs --since 30s crowdsec 2>&1 | grep -Eic 'fatal|unable to run local API|failed to get docker info|lookup socket-proxy|server misbehaving|mapping value is not allowed' || true)\"\n79: echo \"LAPI_TRY=$n STATE=$state HEALTH=$health LAPI_RC=${lapi_rc:-NA} FATAL_COUNT_30S=$fatal\"\n83: && test \"$health\" = \"200\" \\\n126:remove(\"/opt/stacks/crowdsec/config/config.yaml\")\n127:remove(\"/opt/stacks/crowdsec/config/user.yaml\")\n134:p = Path(\"/opt/stacks/crowdsec/config/config.yaml\")\n165: \" credentials_path: /etc/crowdsec/online_api_credentials.yaml\",\n177: crowdsec:\n182: NO_PROXY: \"[IP],localhost,[IP],[IPV6],socket-proxy,crowdsec\"\n221:collect_netbird_peers() {\n222: echo \"NETBIRD_STATUS\"\n223: if ! command -v netbird >/dev/null 2>&1; then\n224: echo \"NETBIRD_CLI_MISSING\"\n228: netbird status 2>&1 | redact | tee /tmp/771l_netbird_status.txt || true\n229: netbird status --json >/tmp/771l_netbird_status.json 2>/tmp/771l_netbird_status_json.err || true\n230: netbird ip 2>/tmp/771l_netbird_ip.err | tee /tmp/771l_netbird_ip.txt || true\n232: echo \"NETBIRD_INTERFACES\"\n233: ip -o -4 addr show | grep -Ei 'wt|netbird|wg|100\\.' || true\n235: python3 - <<'PY' >/tmp/771l_netbird_peers.tsv\n240:for p in [Path(\"/tmp/771l_netbird_ip.txt\")]:\n244:text = Path(\"/tmp/771l_netbird_status.txt\").read_text(errors=\"ignore\") if Path(\"/tmp/771l_netbird_status.txt\").exists() else \"\"\nDEPENDENCY_FILE|path=/root/771l_crowdsec_capi_netbird_vps_egress.sh|mtime=2026-07-02 18:10:45.295356498 +0300|sha256=401396a25d4a5cfa487f67b355b45b27140ed1ac6b49b41e342306cb618dfdb5\n8:PROOF=\"/root/evidence/771L_CROWDSEC_CAPI_NETBIRD_VPS_EGRESS_${TS}_PROOF.txt\"\n10:REMOTE_SCRIPT=\"/tmp/771l_crowdsec_capi_netbird_vps_egress_${TS}.sh\"\n11:LOCAL_REMOTE_SCRIPT=\"/root/771l_crowdsec_capi_netbird_vps_egress_remote_${TS}.sh\"\n18:STACK=/opt/stacks/crowdsec\n20:BACKUP_DIR=\"$STACK/manual-backups/771l-netbird-vps-egress-$TS\"\n21:HEALTH_DIR=/var/lib/homelab-health\n22:HEALTH_FILE=\"$HEALTH_DIR/crowdsec-capi.txt\"\n23:EGRESS_DIR=/etc/homelab-crowdsec-capi-netbird-egress\n24:EGRESS_SERVICE=homelab-crowdsec-capi-netbird-egress.service\n27:mkdir -p \"$BACKUP_DIR\" \"$HEALTH_DIR\" \"$EGRESS_DIR/profiles\"\n33:write_health() {\n39: echo \"CHECK=crowdsec-capi\"\n42: } > \"$HEALTH_FILE\"\n46: docker network inspect crowdsec_default --format '{{(index .IPAM.Config 0).Gateway}}' 2>/dev/null || echo \"[IP]\"\n54: crowdsec:\n79: docker compose up -d --force-recreate crowdsec\n87: state=\"$(docker inspect crowdsec --format 'Running={{.State.Running}} Restarting={{.State.Restarting}} RestartCount={{.RestartCount}} StartedAt={{.State.StartedAt}} ExitCode={{.State.ExitCode}}' 2>/dev/null || true)\"\n88: health=\"$(curl -sk --connect-timeout 3 --max-time 6 -o /tmp/771l_health.html -w '%{http_code}' http://[IP]:8088/health || true)\"\n91: fatal=\"$(docker logs --since 30s crowdsec 2>&1 | grep -Eic 'fatal|unable to run local API|failed to get docker info|lookup socket-proxy|server misbehaving|mapping value is not allowed' || true)\"\n92: echo \"LAPI_TRY=$n STATE=$state HEALTH=$health LAPI_RC=${lapi_rc:-NA} FATAL_COUNT_30S=$fatal\"\n96: && test \"$health\" = \"200\" \\\n139:remove(\"/opt/stacks/crowdsec/config/config.yaml\")\n140:remove(\"/opt/stacks/crowdsec/config/user.yaml\")\n147:p = Path(\"/opt/stacks/crowdsec/config/config.yaml\")\n178: \" credentials_path: /etc/crowdsec/online_api_credentials.yaml\",\n190: crowdsec:\n195: NO_PROXY: \"[IP],localhost,[IP],[IPV6],socket-proxy,crowdsec\"\n234:collect_netbird_peers() {\n235: echo \"NETBIRD_STATUS\"\n236: if ! command -v netbird >/dev/null 2>&1; then\n237: echo \"NETBIRD_CLI_MISSING\"\n241: netbird status 2>&1 | redact | tee /tmp/771l_netbird_status.txt || true\n242: netbird status --json >/tmp/771l_netbird_status.json 2>/tmp/771l_netbird_status_json.err || true\n243: netbird ip 2>/tmp/771l_netbird_ip.err | tee /tmp/771l_netbird_ip.txt || true\n245: echo \"NETBIRD_INTERFACES\"\n246: ip -o -4 addr show | grep -Ei 'wt|netbird|wg|100\\.' || true\nDEPENDENCY_FILE|path=/root/771m_purge_warp_fix_netbird_egress.sh|mtime=2026-07-02 18:16:37.903239779 +0300|sha256=0ae3b0e1e8016da55cf1756e868c51a23fe12328fae6ef4baa61c50b9e48e715\n8:PROOF=\"/root/evidence/771M_PURGE_WARP_FIX_NETBIRD_EGRESS_${TS}_PROOF.txt\"\n10:REMOTE_SCRIPT=\"/tmp/771m_purge_warp_fix_netbird_${TS}.sh\"\n11:LOCAL_REMOTE_SCRIPT=\"/root/771m_purge_warp_fix_netbird_remote_${TS}.sh\"\n18:STACK=/opt/stacks/crowdsec\n20:BACKUP_DIR=\"$STACK/manual-backups/771m-purge-warp-netbird-egress-$TS\"\n21:HEALTH_DIR=/var/lib/homelab-health\n22:HEALTH_FILE=\"$HEALTH_DIR/crowdsec-capi.txt\"\n23:EGRESS_DIR=/etc/homelab-crowdsec-capi-netbird-egress\n24:EGRESS_SERVICE=homelab-crowdsec-capi-netbird-egress.service\n28:mkdir -p \"$BACKUP_DIR\" \"$HEALTH_DIR\" \"$EGRESS_DIR/profiles\"\n34:write_health() {\n40: echo \"CHECK=crowdsec-capi\"\n43: } > \"$HEALTH_FILE\"\n47: docker network inspect crowdsec_default --format '{{(index .IPAM.Config 0).Gateway}}' 2>/dev/null || echo \"[IP]\"\n55: crowdsec:\n80: docker compose up -d --force-recreate crowdsec\n88: state=\"$(docker inspect crowdsec --format 'Running={{.State.Running}} Restarting={{.State.Restarting}} RestartCount={{.RestartCount}} StartedAt={{.State.StartedAt}} ExitCode={{.State.ExitCode}}' 2>/dev/null || true)\"\n89: health=\"$(curl -sk --connect-timeout 3 --max-time 6 -o /tmp/771m_health.html -w '%{http_code}' http://[IP]:8088/health || true)\"\n92: fatal=\"$(docker logs --since 30s crowdsec 2>&1 | grep -Eic 'fatal|unable to run local API|failed to get docker info|lookup socket-proxy|server misbehaving|mapping value is not allowed' || true)\"\n93: echo \"LAPI_TRY=$n STATE=$state HEALTH=$health LAPI_RC=${lapi_rc:-NA} FATAL_COUNT_30S=$fatal\"\n97: && test \"$health\" = \"200\" \\\n140:remove(\"/opt/stacks/crowdsec/config/config.yaml\")\n141:remove(\"/opt/stacks/crowdsec/config/user.yaml\")\n148:p = Path(\"/opt/stacks/crowdsec/config/config.yaml\")\n179: \" credentials_path: /etc/crowdsec/online_api_credentials.yaml\",\n191: crowdsec:\n196: NO_PROXY: \"[IP],localhost,[IP],[IPV6],socket-proxy,crowdsec\"\n258:collect_netbird_peers() {\n259: echo \"NETBIRD_STATUS\"\n260: if ! command -v netbird >/dev/null 2>&1; then\n261: echo \"NETBIRD_CLI_MISSING\"\n265: netbird status 2>&1 | redact | tee /tmp/771m_netbird_status.txt || true\n266: netbird status --json >/tmp/771m_netbird_status.json 2>/tmp/771m_netbird_status_json.err || true\n267: netbird ip 2>/tmp/771m_netbird_ip.err | tee /tmp/771m_netbird_ip.txt || true\n269: echo \"NETBIRD_INTERFACES\"\n270: ip -o -4 addr show | grep -Ei 'wt|netbird|wg|100\\.' || true\nDEPENDENCY_FILE|path=/root/771m_purge_warp_fix_netbird_remote_20260702T151637Z.sh|mtime=2026-07-02 18:16:37.909445276 +0300|sha256=941f683d148a01d9af9ef6ce938c8ee6874d7af1ac1ac5ee96c7d9ca7cc262a4\n5:STACK=/opt/stacks/crowdsec\n7:BACKUP_DIR=\"$STACK/manual-backups/771m-purge-warp-netbird-egress-$TS\"\n8:HEALTH_DIR=/var/lib/homelab-health\n9:HEALTH_FILE=\"$HEALTH_DIR/crowdsec-capi.txt\"\n10:EGRESS_DIR=/etc/homelab-crowdsec-capi-netbird-egress\n11:EGRESS_SERVICE=homelab-crowdsec-capi-netbird-egress.service\n15:mkdir -p \"$BACKUP_DIR\" \"$HEALTH_DIR\" \"$EGRESS_DIR/profiles\"\n21:write_health() {\n27: echo \"CHECK=crowdsec-capi\"\n30: } > \"$HEALTH_FILE\"\n34: docker network inspect crowdsec_default --format '{{(index .IPAM.Config 0).Gateway}}' 2>/dev/null || echo \"[IP]\"\n42: crowdsec:\n67: docker compose up -d --force-recreate crowdsec\n75: state=\"$(docker inspect crowdsec --format 'Running={{.State.Running}} Restarting={{.State.Restarting}} RestartCount={{.RestartCount}} StartedAt={{.State.StartedAt}} ExitCode={{.State.ExitCode}}' 2>/dev/null || true)\"\n76: health=\"$(curl -sk --connect-timeout 3 --max-time 6 -o /tmp/771m_health.html -w '%{http_code}' http://[IP]:8088/health || true)\"\n79: fatal=\"$(docker logs --since 30s crowdsec 2>&1 | grep -Eic 'fatal|unable to run local API|failed to get docker info|lookup socket-proxy|server misbehaving|mapping value is not allowed' || true)\"\n80: echo \"LAPI_TRY=$n STATE=$state HEALTH=$health LAPI_RC=${lapi_rc:-NA} FATAL_COUNT_30S=$fatal\"\n84: && test \"$health\" = \"200\" \\\n127:remove(\"/opt/stacks/crowdsec/config/config.yaml\")\n128:remove(\"/opt/stacks/crowdsec/config/user.yaml\")\n135:p = Path(\"/opt/stacks/crowdsec/config/config.yaml\")\n166: \" credentials_path: /etc/crowdsec/online_api_credentials.yaml\",\n178: crowdsec:\n183: NO_PROXY: \"[IP],localhost,[IP],[IPV6],socket-proxy,crowdsec\"\n245:collect_netbird_peers() {\n246: echo \"NETBIRD_STATUS\"\n247: if ! command -v netbird >/dev/null 2>&1; then\n248: echo \"NETBIRD_CLI_MISSING\"\n252: netbird status 2>&1 | redact | tee /tmp/771m_netbird_status.txt || true\n253: netbird status --json >/tmp/771m_netbird_status.json 2>/tmp/771m_netbird_status_json.err || true\n254: netbird ip 2>/tmp/771m_netbird_ip.err | tee /tmp/771m_netbird_ip.txt || true\n256: echo \"NETBIRD_INTERFACES\"\n257: ip -o -4 addr show | grep -Ei 'wt|netbird|wg|100\\.' || true\n259: python3 - <<'PY' >/tmp/771m_netbird_peers.tsv\n264:p = Path(\"/tmp/771m_netbird_ip.txt\")\n278:jp = Path(\"/tmp/771m_netbird_status.json\")\nDEPENDENCY_FILE|path=/root/771n_netbird_usa_moldova_egress_only.sh|mtime=2026-07-02 18:21:39.746368388 +0300|sha256=420d835318960ae2644dafc93e0427584505acaffa9786927623aeb59ba6ac0e\n8:PROOF=\"/root/evidence/771N_NETBIRD_USA_MOLDOVA_EGRESS_ONLY_${TS}_PROOF.txt\"\n10:REMOTE_SCRIPT=\"/tmp/771n_netbird_usa_moldova_egress_${TS}.sh\"\n11:LOCAL_REMOTE_SCRIPT=\"/root/771n_netbird_usa_moldova_egress_remote_${TS}.sh\"\n18:STACK=/opt/stacks/crowdsec\n20:EGRESS_DIR=/etc/homelab-crowdsec-capi-netbird-egress\n21:HEALTH_DIR=/var/lib/homelab-health\n22:HEALTH_FILE=\"$HEALTH_DIR/crowdsec-capi.txt\"\n25:mkdir -p \"$EGRESS_DIR/profiles\" \"$HEALTH_DIR\"\n32: docker network inspect crowdsec_default --format '{{(index .IPAM.Config 0).Gateway}}' 2>/dev/null || echo \"[IP]\"\n35:write_health() {\n41: echo \"CHECK=crowdsec-capi\"\n44: } > \"$HEALTH_FILE\"\n47:force_crowdsec_safe() {\n51: crowdsec:\n74: docker compose up -d crowdsec\n84: cat >\"$EGRESS_DIR/profiles/${profile}.env\" </etc/systemd/system/homelab-crowdsec-capi-netbird-egress.service <<'EOF'\n95:Description=Homelab CrowdSec CAPI NetBird VPS SOCKS egress\n96:After=network-online.target docker.service netbird.service\n102:EnvironmentFile=/etc/homelab-crowdsec-capi-netbird-egress/current.env\n111: cat >/usr/local/sbin/homelab-crowdsec-capi-egress-switch <<'EOF'\n115:base=/etc/homelab-crowdsec-capi-netbird-egress\n123:systemctl restart homelab-crowdsec-capi-netbird-egress.service\n126:systemctl --no-pager --full status homelab-crowdsec-capi-netbird-egress.service | sed -n '1,30p'\n128: chmod 755 /usr/local/sbin/homelab-crowdsec-capi-egress-switch\n131: systemctl enable --now homelab-crowdsec-capi-netbird-egress.service\n132: systemctl restart homelab-crowdsec-capi-netbird-egress.service\n134: systemctl is-active homelab-crowdsec-capi-netbird-egress.service || true\n146: if \"HOMELAB_771\" in line and \"CROWDSEC_CAPI\" in line and \"BEGIN\" in line:\n149: if \"HOMELAB_771\" in line and \"CROWDSEC_CAPI\" in line and \"END\" in line:\n160:out.append(\"# HOMELAB_771N_CROWDSEC_CAPI_NETBIRD_BEGIN\")\n163:out.append(\"# HOMELAB_771N_CROWDSEC_CAPI_NETBIRD_END\")\n194: code=\"$(curl -sk --proxy \"socks5h://[IP]:${port}\" --connect-timeout 12 --max-time 45 -o /tmp/771n_capi_${profile}.body -w '%{http_code}' https://api.crowdsec.net/v3/watchers/login || true)\"\n203: echo \"VPS_EGRESS_OK profile=$profile user=$user ip=$ip\"\n214: echo \"VPS_EGRESS_NOT_READY profile=$profile ip=$ip\"\nDEPENDENCY_FILE|path=/root/771n_netbird_usa_moldova_egress_remote_20260702T152139Z.sh|mtime=2026-07-02 18:21:39.750535800 +0300|sha256=1c25cbc13f524d0f6974a71c255c5634c69380818cb1b5ad4f8f6ea9bf8c6a01\n5:STACK=/opt/stacks/crowdsec\n7:EGRESS_DIR=/etc/homelab-crowdsec-capi-netbird-egress\n8:HEALTH_DIR=/var/lib/homelab-health\n9:HEALTH_FILE=\"$HEALTH_DIR/crowdsec-capi.txt\"\n12:mkdir -p \"$EGRESS_DIR/profiles\" \"$HEALTH_DIR\"\n19: docker network inspect crowdsec_default --format '{{(index .IPAM.Config 0).Gateway}}' 2>/dev/null || echo \"[IP]\"\n22:write_health() {\n28: echo \"CHECK=crowdsec-capi\"\n31: } > \"$HEALTH_FILE\"\n34:force_crowdsec_safe() {\n38: crowdsec:\n61: docker compose up -d crowdsec\n71: cat >\"$EGRESS_DIR/profiles/${profile}.env\" </etc/systemd/system/homelab-crowdsec-capi-netbird-egress.service <<'EOF'\n82:Description=Homelab CrowdSec CAPI NetBird VPS SOCKS egress\n83:After=network-online.target docker.service netbird.service\n89:EnvironmentFile=/etc/homelab-crowdsec-capi-netbird-egress/current.env\n98: cat >/usr/local/sbin/homelab-crowdsec-capi-egress-switch <<'EOF'\n102:base=/etc/homelab-crowdsec-capi-netbird-egress\n110:systemctl restart homelab-crowdsec-capi-netbird-egress.service\n113:systemctl --no-pager --full status homelab-crowdsec-capi-netbird-egress.service | sed -n '1,30p'\n115: chmod 755 /usr/local/sbin/homelab-crowdsec-capi-egress-switch\n118: systemctl enable --now homelab-crowdsec-capi-netbird-egress.service\n119: systemctl restart homelab-crowdsec-capi-netbird-egress.service\n121: systemctl is-active homelab-crowdsec-capi-netbird-egress.service || true\n133: if \"HOMELAB_771\" in line and \"CROWDSEC_CAPI\" in line and \"BEGIN\" in line:\n136: if \"HOMELAB_771\" in line and \"CROWDSEC_CAPI\" in line and \"END\" in line:\n147:out.append(\"# HOMELAB_771N_CROWDSEC_CAPI_NETBIRD_BEGIN\")\n150:out.append(\"# HOMELAB_771N_CROWDSEC_CAPI_NETBIRD_END\")\n181: code=\"$(curl -sk --proxy \"socks5h://[IP]:${port}\" --connect-timeout 12 --max-time 45 -o /tmp/771n_capi_${profile}.body -w '%{http_code}' https://api.crowdsec.net/v3/watchers/login || true)\"\n190: echo \"VPS_EGRESS_OK profile=$profile user=$user ip=$ip\"\n201: echo \"VPS_EGRESS_NOT_READY profile=$profile ip=$ip\"\n206:echo \"STEP=771N_NETBIRD_USA_MOLDOVA_EGRESS_ONLY\"\n207:echo \"RULE=WARP_PURGED_NETBIRD_ONLY\"\n215:echo \"CROWDSEC_SAFE_STATE\"\nDEPENDENCY_FILE|path=/root/771o_netbird_management_acl_discovery.sh|mtime=2026-07-02 18:24:39.066582039 +0300|sha256=50526afeaeb99fa9a8018374abe82731dcb9e93eac4be64934d0c6a8610be322\n8:PROOF=\"/root/evidence/771O_NETBIRD_MANAGEMENT_ACL_DISCOVERY_${TS}_PROOF.txt\"\n14: echo \"STEP=771O_NETBIRD_MANAGEMENT_ACL_DISCOVERY\"\n17: echo \"RULE=WARP_PURGED_NETBIRD_POLICY_DISCOVERY_ONLY\"\n24: echo \"EDGE_NETBIRD_ACL_DIAG\"\n31: echo NETBIRD_STATUS\n34: echo NETBIRD_JSON_TARGETS\n35: netbird status --json >/tmp/771o_netbird_status.json 2>/tmp/771o_netbird_status_json.err || true\n39:p=Path(\\\"/tmp/771o_netbird_status.json\\\")\n42: print(Path(\\\"/tmp/771o_netbird_status_json.err\\\").read_text(errors=\\\"ignore\\\")[:500] if Path(\\\"/tmp/771o_netbird_status_json.err\\\").exists() else \\\"\\\")\n46:names=(\\\"e3qxxx\\\",\\\"183-106\\\",\\\"moldova\\\",\\\"usa\\\")\n82: echo CROWDSEC_SAFE_STATE\n83: sudo docker ps --format \\\"{{.Names}} {{.Status}} {{.Ports}}\\\" | grep -E \\\"^(crowdsec|socket-proxy|npmplus)\\\" || true\n87: echo \"DISCOVER_NETBIRD_MANAGEMENT_ON_KNOWN_HOSTS\"\n101: echo DOCKER_NETBIRD_CONTAINERS\n102: docker ps -a --format \"{{.Names}} {{.Image}} {{.Status}} {{.Ports}}\" 2>/dev/null | grep -Ei \"netbird|management|signal|relay|coturn|zitadel|dashboard\" || true\n103: echo SYSTEMD_NETBIRD\n104: systemctl list-units --all --no-pager 2>/dev/null | grep -Ei \"netbird|signal|relay|coturn|zitadel\" || true\n106: find /opt /etc /var/lib -maxdepth 5 \\( -iname \"*netbird*\" -o -iname \"*management*.json\" -o -iname \"store.json\" \\) 2>/dev/null | sed -n \"1,120p\"\n111: echo DOCKER_NETBIRD_CONTAINERS\n112: docker ps -a --format \\\"{{.Names}} {{.Image}} {{.Status}} {{.Ports}}\\\" 2>/dev/null | grep -Ei \\\"netbird|management|signal|relay|coturn|zitadel|dashboard\\\" || true\n113: echo SYSTEMD_NETBIRD\n114: systemctl list-units --all --no-pager 2>/dev/null | grep -Ei \\\"netbird|signal|relay|coturn|zitadel\\\" || true\n116: find /opt /etc /var/lib -maxdepth 5 \\( -iname \\\"*netbird*\\\" -o -iname \\\"*management*.json\\\" -o -iname \\\"store.json\\\" \\) 2>/dev/null | sed -n \\\"1,120p\\\"\n122: echo \"TRY_PARSE_NETBIRD_STORE_IF_FOUND_LOCALLY\"\n124: candidates=$(find /opt /etc /var/lib -maxdepth 7 \\( -path \"*netbird*\" -a \\( -iname \"store.json\" -o -iname \"*.db\" -o -iname \"*.json\" \\) \\) 2>/dev/null | sed -n \"1,40p\")\n163: echo \"STATUS=OK_771O_DISCOVERY_DONE_NEXT_FIX_NETBIRD_POLICY_OR_VPS_AGENT\"\nDEPENDENCY_FILE|path=/root/771p_netbird_vps_peer_repair.sh|mtime=2026-07-02 19:32:10.024856543 +0300|sha256=f37aac25cac5b1c3983392070ddaff09e28497efa28884b03d23efc09a825747\n8:PROOF=\"/root/evidence/771P_NETBIRD_VPS_PEER_REPAIR_${TS}_PROOF.txt\"\n11:MOLDOVA_NB=[IP]\n16: echo \"STEP=771P_NETBIRD_VPS_PEER_REPAIR\"\n19: echo \"RULE=NO_WARP_FIX_NETBIRD_USA_MOLDOVA_PEERS_FOR_CROWDSEC_CAPI\"\n23: echo \"CONFIG_CHANGE=YES_RESTART_NETBIRD_ON_VPS_IF_PUBLIC_SSH_FOUND\"\n26: echo \"LOCAL_PVE01_NETBIRD_STATUS\"\n29: for ip in \"$USA_NB\" \"$MOLDOVA_NB\"; do\n35: echo \"EDGE_NETBIRD_STATUS_AND_TARGETS\"\n42: echo NETBIRD_STATUS\n46: netbird status --json >/tmp/771p_edge_nb.json 2>/tmp/771p_edge_nb.err || true\n57: ip=peer.get(\\\"netbirdIp\\\",\\\"\\\")\n59: print(\\\"PEER\\\", fqdn, ip, \\\"status=\\\"+str(peer.get(\\\"status\\\")), \\\"handshake=\\\"+str(peer.get(\\\"lastWireguardHandshake\\\")), \\\"relay=\\\"+str(peer.get(\\\"relayAddress\\\")), \\\"conn=\\\"+str(peer.get(\\\"connectionType\\\")))\n80: grep -nEi 'e3qxxx|183-106|moldova|молдов|usa|america|vps|netbird|alexhost|aeza|hetzner|public ip|external ip|ssh' /etc/pve/31_HOMELAB_REFERENCE.md 2>/dev/null | sed -n '1,260p' || true\n81: echo \"SOURCE=systemd_netbird_vps\"\n84: grep -RHEi 'Host |HostName|User |Port |e3qxxx|183-106|moldova|usa|vps' [SENSITIVE_PATH] /etc/ssh/ssh_config /etc/ssh/ssh_config.d/* 2>/dev/null | sed -n '1,260p' || true\n110: if any(x in low for x in [\"183-106\", \"moldova\", \"молдов\", \"chisinau\"]):\n111: profile = \"moldova\"\n123: if \"netbird.selfhosted\" in host:\n138: echo \"TEST_PUBLIC_SSH_AND_RESTART_NETBIRD_ON_VPS_IF_FOUND\"\n159: systemctl restart netbird 2>/dev/null || true\n173: echo \"CROWDSEC_SAFE_POSTCHECK\"\n175: sudo docker ps --format \\\"{{.Names}} {{.Status}} {{.Ports}}\\\" | grep -E \\\"^(crowdsec|socket-proxy|npmplus)\\\" || true\n182: echo \"STATUS=OK_771P_VPS_PUBLIC_ACCESS_FOUND_RESTARTED_NETBIRD_REVIEW_RECHECK\"\nDEPENDENCY_FILE|path=/root/771q_netbird_public_vps_repair_then_capi.sh|mtime=2026-07-02 19:45:41.256069566 +0300|sha256=0b6e42e5b4013f7ee2191b140c5c507877fb05e2a4fdd7b39bea7b7c6b108071\n8:PROOF=\"/root/evidence/771Q_NETBIRD_PUBLIC_VPS_REPAIR_THEN_CAPI_${TS}_PROOF.txt\"\n11:MOLDOVA_NB=[IP]\n16: echo \"STEP=771Q_NETBIRD_PUBLIC_VPS_REPAIR_THEN_CAPI\"\n19: echo \"RULE=NO_WARP_REPAIR_NETBIRD_USA_MOLDOVA_THEN_CROWDSEC_CAPI\"\n23: echo \"CONFIG_CHANGE=YES_RESTART_PUBLIC_VPS_NETBIRD_AND_CAPI_IF_ROUTE_READY\"\n38: \"echo SSH_OK; echo HOSTNAME=\\$(hostname); command -v netbird >/dev/null 2>&1 && netbird status 2>&1 || echo NETBIRD_CLI_MISSING; curl -4 -sk --connect-timeout 8 --max-time 20 -o /tmp/771q_capi.body -w 'DIRECT_CAPI_HTTP=%{http_code}\\n' https://api.crowdsec.net/v3/watchers/login || true\" \\\n58: \"moldova|debian|netbird.vpn.gram1.ru\" \\\n59: \"moldova|root|netbird.vpn.gram1.ru\" \\\n60: \"moldova|debian|[IP]\" \\\n61: \"moldova|root|[IP]\" \\\n62: \"netbird-ui|debian|nb.pvepro.ru\" \\\n63: \"netbird-ui|root|nb.pvepro.ru\"\n74: echo \"RESTART_NETBIRD_ON_REAL_PUBLIC_VPS_ONLY\"\n79: netbird.vpn.gram1.ru|[IP]|nb.pvepro.ru)\n87: \"echo BEFORE_HOSTNAME=\\$(hostname); command -v netbird >/dev/null 2>&1 && netbird status 2>&1 || true; (sudo systemctl restart netbird 2>/dev/null || systemctl restart netbird 2>/dev/null || true); sleep 15; echo AFTER; command -v netbird >/dev/null 2>&1 && netbird status 2>&1 || true; curl -4 -sk --connect-timeout 8 --max-time 20 -o /tmp/771q_capi_after.body -w 'DIRECT_CAPI_HTTP_AFTER=%{http_code}\\n' https://api.crowdsec.net/v3/watchers/login || true\" \\\n106: echo \"RECHECK_EDGE_USA_MOLDOVA_AFTER_PUBLIC_RESTART\"\n109: systemctl restart netbird 2>/dev/null || true\n112: netbird status --json >/tmp/771q_edge_nb.json 2>/tmp/771q_edge_nb.err || true\n121: ip=peer.get(\\\"netbirdIp\\\",\\\"\\\")\n123: print(\\\"PEER\\\", fqdn, ip, \\\"status=\\\"+str(peer.get(\\\"status\\\")), \\\"handshake=\\\"+str(peer.get(\\\"lastWireguardHandshake\\\")), \\\"relay=\\\"+str(peer.get(\\\"relayAddress\\\")), \\\"conn=\\\"+str(peer.get(\\\"connectionType\\\")))\n139: echo \"EDGE_BUILD_NETBIRD_EGRESS_AND_REGISTER_CAPI_IF_REACHABLE\"\n144:STACK=/opt/stacks/crowdsec\n146:EGRESS_DIR=/etc/homelab-crowdsec-capi-netbird-egress\n147:HEALTH_DIR=/var/lib/homelab-health\n148:HEALTH_FILE=\"$HEALTH_DIR/crowdsec-capi.txt\"\n151:mkdir -p \"$EGRESS_DIR/profiles\" \"$HEALTH_DIR\"\n158: docker network inspect crowdsec_default --format '{{(index .IPAM.Config 0).Gateway}}' 2>/dev/null || echo \"[IP]\"\n161:write_health() {\n167: echo \"CHECK=crowdsec-capi\"\n170: } > \"$HEALTH_FILE\"\n177: crowdsec:\n198: docker compose up -d crowdsec\n206: health=\"$(curl -sk --connect-timeout 3 --max-time 6 -o /tmp/771q_health.html -w '%{http_code}' http://[IP]:8088/health || true)\"\n209: echo \"LAPI_TRY=$n HEALTH=$health LAPI_RC=${rc:-NA}\"\n211: test \"$health\" = \"200\" && test \"${rc:-NA}\" = \"0\" && return 0\n241:remove(\"/opt/stacks/crowdsec/config/config.yaml\")\nDEPENDENCY_FILE|path=/root/771r_edge_netbird_egress_capi_20260702T165517Z.sh|mtime=2026-07-02 19:55:17.909628261 +0300|sha256=9c6f911768869c984ec41016b3134be75620100fac8c60a8b21ef6bde58c6868\n5:STACK=/opt/stacks/crowdsec\n7:EGRESS_DIR=/etc/homelab-crowdsec-capi-netbird-egress\n8:HEALTH_DIR=/var/lib/homelab-health\n9:HEALTH_FILE=\"$HEALTH_DIR/crowdsec-capi.txt\"\n12:mkdir -p \"$EGRESS_DIR/profiles\" \"$HEALTH_DIR\"\n19: docker network inspect crowdsec_default --format '{{(index .IPAM.Config 0).Gateway}}' 2>/dev/null || echo \"[IP]\"\n22:write_health() {\n29: echo \"CHECK=crowdsec-capi\"\n32: } > \"$HEALTH_FILE\"\n39: crowdsec:\n62: docker compose up -d crowdsec\n66: local label n health out rc fatal\n71: health=\"$(curl -sk --connect-timeout 3 --max-time 6 -o /tmp/771r_health.html -w '%{http_code}' http://[IP]:8088/health || true)\"\n74: fatal=\"$(docker logs --since 30s crowdsec 2>&1 | grep -Eic 'fatal|unable to run local API|failed to get docker info|lookup socket-proxy|server misbehaving|mapping value is not allowed' || true)\"\n75: echo \"LAPI_TRY=$n HEALTH=$health LAPI_RC=${rc:-NA} FATAL_COUNT_30S=$fatal\"\n77: if test \"$health\" = \"200\" && test \"${rc:-NA}\" = \"0\" && test \"$fatal\" = \"0\"; then\n114:remove(\"/opt/stacks/crowdsec/config/config.yaml\")\n115:remove(\"/opt/stacks/crowdsec/config/user.yaml\")\n122:p = Path(\"/opt/stacks/crowdsec/config/config.yaml\")\n153: \" credentials_path: /etc/crowdsec/online_api_credentials.yaml\",\n166: crowdsec:\n171: NO_PROXY: \"[IP],localhost,[IP],[IPV6],socket-proxy,crowdsec\"\n196:test_target_ssh_egress() {\n202: echo \"TEST_TARGET_SSH_EGRESS profile=$profile ip=$ip\"\n212: \"echo SSH_OK; hostname; curl -4 -sk --connect-timeout 8 --max-time 20 -o /tmp/771r_capi.body -w 'DIRECT_CAPI_HTTP=%{http_code}\\n' https://api.crowdsec.net/v3/watchers/login || true\" \\\n234: code=\"$(curl -sk --proxy \"socks5h://[IP]:${port}\" --connect-timeout 12 --max-time 45 -o /tmp/771r_capi_proxy.body -w '%{http_code}' https://api.crowdsec.net/v3/watchers/login || true)\"\n255:install_egress_service() {\n261: echo \"INSTALL_EGRESS profile=$profile user=$user ip=$ip bridge=$b\"\n263: mkdir -p \"$EGRESS_DIR/profiles\"\n264: cat >\"$EGRESS_DIR/profiles/${profile}.env\" </etc/systemd/system/homelab-crowdsec-capi-netbird-egress.service <<'EOF'\n275:Description=Homelab CrowdSec CAPI NetBird VPS SOCKS egress\n276:After=network-online.target docker.service netbird.service\n282:EnvironmentFile=/etc/homelab-crowdsec-capi-netbird-egress/current.env\n291: cat >/usr/local/sbin/homelab-crowdsec-capi-egress-switch <<'EOF'\nDEPENDENCY_FILE|path=/root/771r_netbird_public_vps_fixed_then_capi.sh|mtime=2026-07-02 19:55:17.904566554 +0300|sha256=dc8119b6815d60e08f442e77faba6ce6e9899c4120d78490886e222b8510efa7\n8:PROOF=\"/root/evidence/771R_NETBIRD_PUBLIC_VPS_FIXED_THEN_CAPI_${TS}_PROOF.txt\"\n10:EDGE_REMOTE=\"/tmp/771r_edge_netbird_egress_capi_${TS}.sh\"\n11:EDGE_LOCAL=\"/root/771r_edge_netbird_egress_capi_${TS}.sh\"\n31: \"echo SSH_OK; echo USER=\\$(id -un); echo HOSTNAME=\\$(hostname); command -v netbird >/dev/null 2>&1 && netbird status 2>&1 || echo NETBIRD_CLI_MISSING; curl -4 -sk --connect-timeout 8 --max-time 20 -o /tmp/771r_capi.body -w 'DIRECT_CAPI_HTTP=%{http_code}\\n' https://api.crowdsec.net/v3/watchers/login || true\" \\\n49:STACK=/opt/stacks/crowdsec\n51:EGRESS_DIR=/etc/homelab-crowdsec-capi-netbird-egress\n52:HEALTH_DIR=/var/lib/homelab-health\n53:HEALTH_FILE=\"$HEALTH_DIR/crowdsec-capi.txt\"\n56:mkdir -p \"$EGRESS_DIR/profiles\" \"$HEALTH_DIR\"\n63: docker network inspect crowdsec_default --format '{{(index .IPAM.Config 0).Gateway}}' 2>/dev/null || echo \"[IP]\"\n66:write_health() {\n73: echo \"CHECK=crowdsec-capi\"\n76: } > \"$HEALTH_FILE\"\n83: crowdsec:\n106: docker compose up -d crowdsec\n110: local label n health out rc fatal\n115: health=\"$(curl -sk --connect-timeout 3 --max-time 6 -o /tmp/771r_health.html -w '%{http_code}' http://[IP]:8088/health || true)\"\n118: fatal=\"$(docker logs --since 30s crowdsec 2>&1 | grep -Eic 'fatal|unable to run local API|failed to get docker info|lookup socket-proxy|server misbehaving|mapping value is not allowed' || true)\"\n119: echo \"LAPI_TRY=$n HEALTH=$health LAPI_RC=${rc:-NA} FATAL_COUNT_30S=$fatal\"\n121: if test \"$health\" = \"200\" && test \"${rc:-NA}\" = \"0\" && test \"$fatal\" = \"0\"; then\n158:remove(\"/opt/stacks/crowdsec/config/config.yaml\")\n159:remove(\"/opt/stacks/crowdsec/config/user.yaml\")\n166:p = Path(\"/opt/stacks/crowdsec/config/config.yaml\")\n197: \" credentials_path: /etc/crowdsec/online_api_credentials.yaml\",\n210: crowdsec:\n215: NO_PROXY: \"[IP],localhost,[IP],[IPV6],socket-proxy,crowdsec\"\n240:test_target_ssh_egress() {\n246: echo \"TEST_TARGET_SSH_EGRESS profile=$profile ip=$ip\"\n256: \"echo SSH_OK; hostname; curl -4 -sk --connect-timeout 8 --max-time 20 -o /tmp/771r_capi.body -w 'DIRECT_CAPI_HTTP=%{http_code}\\n' https://api.crowdsec.net/v3/watchers/login || true\" \\\n278: code=\"$(curl -sk --proxy \"socks5h://[IP]:${port}\" --connect-timeout 12 --max-time 45 -o /tmp/771r_capi_proxy.body -w '%{http_code}' https://api.crowdsec.net/v3/watchers/login || true)\"\n299:install_egress_service() {\n305: echo \"INSTALL_EGRESS profile=$profile user=$user ip=$ip bridge=$b\"\n307: mkdir -p \"$EGRESS_DIR/profiles\"\n308: cat >\"$EGRESS_DIR/profiles/${profile}.env\" </etc/systemd/system/homelab-crowdsec-capi-netbird-egress.service <<'EOF'\nDEPENDENCY_FILE|path=/root/771s_edge_capi_netbird_20260702T170251Z.sh|mtime=2026-07-02 20:02:51.044349876 +0300|sha256=c4783c14a3a132616af5db6a065270ccd39a7690f9fd38d0ef27fe5de6bd07e5\n6:PROFILE=\"${3:-moldova}\"\n7:STACK=/opt/stacks/crowdsec\n9:EGRESS_DIR=/etc/homelab-crowdsec-capi-netbird-egress\n10:HEALTH_DIR=/var/lib/homelab-health\n11:HEALTH_FILE=\"$HEALTH_DIR/crowdsec-capi.txt\"\n14:mkdir -p \"$EGRESS_DIR/profiles\" \"$HEALTH_DIR\"\n21: docker network inspect crowdsec_default --format '{{(index .IPAM.Config 0).Gateway}}' 2>/dev/null || echo \"[IP]\"\n24:write_health() {\n30: echo \"CHECK=crowdsec-capi\"\n33: } > \"$HEALTH_FILE\"\n40: crowdsec:\n62: docker compose up -d crowdsec\n70: health=\"$(curl -sk --connect-timeout 3 --max-time 6 -o /tmp/771s_health.html -w '%{http_code}' http://[IP]:8088/health || true)\"\n73: fatal=\"$(docker logs --since 30s crowdsec 2>&1 | grep -Eic 'fatal|unable to run local API|failed to get docker info|lookup socket-proxy|server misbehaving|mapping value is not allowed' || true)\"\n74: echo \"LAPI_TRY=$n HEALTH=$health LAPI_RC=${rc:-NA} FATAL_COUNT_30S=$fatal\"\n76: if test \"$health\" = \"200\" && test \"${rc:-NA}\" = \"0\" && test \"$fatal\" = \"0\"; then\n113:remove(\"/opt/stacks/crowdsec/config/config.yaml\")\n114:remove(\"/opt/stacks/crowdsec/config/user.yaml\")\n121:p=Path(\"/opt/stacks/crowdsec/config/config.yaml\")\n153: \" credentials_path: /etc/crowdsec/online_api_credentials.yaml\",\n165: crowdsec:\n170: NO_PROXY: \"[IP],localhost,[IP],[IPV6],socket-proxy,crowdsec\"\n195:test_target_ssh_egress() {\n199: echo \"TEST_TARGET_SSH_EGRESS profile=$profile ip=$ip\"\n209: \"echo SSH_OK; hostname; curl -4 -sk --connect-timeout 8 --max-time 20 -o /tmp/771s_capi.body -w 'DIRECT_CAPI_HTTP=%{http_code}\\n' https://api.crowdsec.net/v3/watchers/login || true\" \\\n231: code=\"$(curl -sk --proxy \"socks5h://[IP]:${port}\" --connect-timeout 12 --max-time 45 -o /tmp/771s_capi_proxy.body -w '%{http_code}' https://api.crowdsec.net/v3/watchers/login || true)\"\n252:install_egress_service() {\n257: echo \"INSTALL_EGRESS profile=$profile user=$user ip=$ip bridge=$b\"\n259: mkdir -p \"$EGRESS_DIR/profiles\"\n260: cat >\"$EGRESS_DIR/profiles/${profile}.env\" </etc/systemd/system/homelab-crowdsec-capi-netbird-egress.service <<'EOF'\n271:Description=Homelab CrowdSec CAPI NetBird VPS SOCKS egress\n272:After=network-online.target docker.service netbird.service\n278:EnvironmentFile=/etc/homelab-crowdsec-capi-netbird-egress/current.env\n287: cat >/usr/local/sbin/homelab-crowdsec-capi-egress-switch <<'EOF'\nDEPENDENCY_FILE|path=/root/771s_vps_identity_repair_netbird_capi.sh|mtime=2026-07-02 20:02:51.038348586 +0300|sha256=16121a810320b1517291830ba128baf6c68e7b3e6f78786481dedb78a1f061b9\n8:PROOF=\"/root/evidence/771S_VPS_IDENTITY_REPAIR_NETBIRD_CAPI_${TS}_PROOF.txt\"\n11:VPS_ALIAS=netbird.vpn.gram1.ru\n13:EDGE_REMOTE=\"/tmp/771s_edge_capi_netbird_${TS}.sh\"\n14:EDGE_LOCAL=\"/root/771s_edge_capi_netbird_${TS}.sh\"\n28:PROFILE=\"${3:-moldova}\"\n29:STACK=/opt/stacks/crowdsec\n31:EGRESS_DIR=/etc/homelab-crowdsec-capi-netbird-egress\n32:HEALTH_DIR=/var/lib/homelab-health\n33:HEALTH_FILE=\"$HEALTH_DIR/crowdsec-capi.txt\"\n36:mkdir -p \"$EGRESS_DIR/profiles\" \"$HEALTH_DIR\"\n43: docker network inspect crowdsec_default --format '{{(index .IPAM.Config 0).Gateway}}' 2>/dev/null || echo \"[IP]\"\n46:write_health() {\n52: echo \"CHECK=crowdsec-capi\"\n55: } > \"$HEALTH_FILE\"\n62: crowdsec:\n84: docker compose up -d crowdsec\n92: health=\"$(curl -sk --connect-timeout 3 --max-time 6 -o /tmp/771s_health.html -w '%{http_code}' http://[IP]:8088/health || true)\"\n95: fatal=\"$(docker logs --since 30s crowdsec 2>&1 | grep -Eic 'fatal|unable to run local API|failed to get docker info|lookup socket-proxy|server misbehaving|mapping value is not allowed' || true)\"\n96: echo \"LAPI_TRY=$n HEALTH=$health LAPI_RC=${rc:-NA} FATAL_COUNT_30S=$fatal\"\n98: if test \"$health\" = \"200\" && test \"${rc:-NA}\" = \"0\" && test \"$fatal\" = \"0\"; then\n135:remove(\"/opt/stacks/crowdsec/config/config.yaml\")\n136:remove(\"/opt/stacks/crowdsec/config/user.yaml\")\n143:p=Path(\"/opt/stacks/crowdsec/config/config.yaml\")\n175: \" credentials_path: /etc/crowdsec/online_api_credentials.yaml\",\n187: crowdsec:\n192: NO_PROXY: \"[IP],localhost,[IP],[IPV6],socket-proxy,crowdsec\"\n217:test_target_ssh_egress() {\n221: echo \"TEST_TARGET_SSH_EGRESS profile=$profile ip=$ip\"\n231: \"echo SSH_OK; hostname; curl -4 -sk --connect-timeout 8 --max-time 20 -o /tmp/771s_capi.body -w 'DIRECT_CAPI_HTTP=%{http_code}\\n' https://api.crowdsec.net/v3/watchers/login || true\" \\\n253: code=\"$(curl -sk --proxy \"socks5h://[IP]:${port}\" --connect-timeout 12 --max-time 45 -o /tmp/771s_capi_proxy.body -w '%{http_code}' https://api.crowdsec.net/v3/watchers/login || true)\"\n274:install_egress_service() {\n279: echo \"INSTALL_EGRESS profile=$profile user=$user ip=$ip bridge=$b\"\n281: mkdir -p \"$EGRESS_DIR/profiles\"\n282: cat >\"$EGRESS_DIR/profiles/${profile}.env\" </etc/systemd/system/homelab-crowdsec-capi-netbird-egress.service <<'EOF'\nDEPENDENCY_FILE|path=/root/771t_temp_hostkey_netbird_identity_then_capi.sh|mtime=2026-07-02 20:10:05.923400398 +0300|sha256=0c3d628b23b67849d52112322ab2e1ee3779012c10578a0b8cece0d8ddf639da\n8:PROOF=\"/root/evidence/771T_TEMP_HOSTKEY_NETBIRD_IDENTITY_THEN_CAPI_${TS}_PROOF.txt\"\n29:STACK=/opt/stacks/crowdsec\n31:EGRESS_DIR=/etc/homelab-crowdsec-capi-netbird-egress\n32:HEALTH_DIR=/var/lib/homelab-health\n33:HEALTH_FILE=\"$HEALTH_DIR/crowdsec-capi.txt\"\n36:mkdir -p \"$EGRESS_DIR/profiles\" \"$HEALTH_DIR\"\n43: docker network inspect crowdsec_default --format '{{(index .IPAM.Config 0).Gateway}}' 2>/dev/null || echo \"[IP]\"\n46:write_health() {\n50: echo \"CHECK=crowdsec-capi\"\n53: } > \"$HEALTH_FILE\"\n60: crowdsec:\n82: docker compose up -d crowdsec\n90: health=\"$(curl -sk --connect-timeout 3 --max-time 6 -o /tmp/771t_health.html -w '%{http_code}' http://[IP]:8088/health || true)\"\n93: fatal=\"$(docker logs --since 30s crowdsec 2>&1 | grep -Eic 'fatal|unable to run local API|failed to get docker info|lookup socket-proxy|server misbehaving|mapping value is not allowed' || true)\"\n94: echo \"LAPI_TRY=$n HEALTH=$health LAPI_RC=${rc:-NA} FATAL_COUNT_30S=$fatal\"\n96: if test \"$health\" = \"200\" && test \"${rc:-NA}\" = \"0\" && test \"$fatal\" = \"0\"; then\n131:remove(\"/opt/stacks/crowdsec/config/config.yaml\")\n132:remove(\"/opt/stacks/crowdsec/config/user.yaml\")\n139:p=Path(\"/opt/stacks/crowdsec/config/config.yaml\")\n169: \" credentials_path: /etc/crowdsec/online_api_credentials.yaml\",\n181: crowdsec:\n186: NO_PROXY: \"[IP],localhost,[IP],[IPV6],socket-proxy,crowdsec\"\n211:install_egress() {\n213: echo \"INSTALL_EGRESS target=$TARGET_NB user=$TARGET_USER profile=$PROFILE bridge=$b\"\n215: echo \"VERIFY_NETBIRD_SSH_TO_TARGET\"\n218: \"echo SSH_OK; hostname; curl -4 -sk --connect-timeout 8 --max-time 20 -o /tmp/771t_capi_direct.body -w 'DIRECT_CAPI_HTTP=%{http_code}\\n' https://api.crowdsec.net/v3/watchers/login || true\" >\"$out\" 2>&1\n226: mkdir -p \"$EGRESS_DIR/profiles\"\n227: cat >\"$EGRESS_DIR/profiles/${PROFILE}.env\" </etc/systemd/system/homelab-crowdsec-capi-netbird-egress.service <<'EOF'\n238:Description=Homelab CrowdSec CAPI NetBird VPS SOCKS egress\n239:After=network-online.target docker.service netbird.service\n245:EnvironmentFile=/etc/homelab-crowdsec-capi-netbird-egress/current.env\n254: cat >/usr/local/sbin/homelab-crowdsec-capi-egress-switch <<'EOF'\n258:base=/etc/homelab-crowdsec-capi-netbird-egress\n266:systemctl restart homelab-crowdsec-capi-netbird-egress.service\nDEPENDENCY_FILE|path=/root/771u_find_key_fix_netbird_egress_capi.sh|mtime=2026-07-02 20:19:23.540526618 +0300|sha256=6b6511ec3d614793e3542777ccfcb5e8c5f09cff77ee2d57624b3ac980787bcb\n8:PROOF=\"/root/evidence/771U_FIND_KEY_FIX_NETBIRD_EGRESS_CAPI_${TS}_PROOF.txt\"\n29:STACK=/opt/stacks/crowdsec\n31:EGRESS_DIR=/etc/homelab-crowdsec-capi-netbird-egress\n32:HEALTH_DIR=/var/lib/homelab-health\n33:HEALTH_FILE=\"$HEALTH_DIR/crowdsec-capi.txt\"\n34:SSH_KEY=\"$EGRESS_DIR/id_ed25519\"\n37:mkdir -p \"$EGRESS_DIR/profiles\" \"$HEALTH_DIR\"\n38:chmod 700 \"$EGRESS_DIR\"\n45: docker network inspect crowdsec_default --format '{{(index .IPAM.Config 0).Gateway}}' 2>/dev/null || echo \"[IP]\"\n48:write_health() {\n52: echo \"CHECK=crowdsec-capi\"\n55: } > \"$HEALTH_FILE\"\n62: crowdsec:\n84: docker compose up -d crowdsec\n92: health=\"$(curl -sk --connect-timeout 3 --max-time 6 -o /tmp/771u_health.html -w '%{http_code}' http://[IP]:8088/health || true)\"\n95: fatal=\"$(docker logs --since 30s crowdsec 2>&1 | grep -Eic 'fatal|unable to run local API|failed to get docker info|lookup socket-proxy|server misbehaving|mapping value is not allowed' || true)\"\n96: echo \"LAPI_TRY=$n HEALTH=$health LAPI_RC=${rc:-NA} FATAL_COUNT_30S=$fatal\"\n98: if test \"$health\" = \"200\" && test \"${rc:-NA}\" = \"0\" && test \"$fatal\" = \"0\"; then\n133:remove(\"/opt/stacks/crowdsec/config/config.yaml\")\n134:remove(\"/opt/stacks/crowdsec/config/user.yaml\")\n141:p=Path(\"/opt/stacks/crowdsec/config/config.yaml\")\n170: \" credentials_path: /etc/crowdsec/online_api_credentials.yaml\",\n182: crowdsec:\n187: NO_PROXY: \"[IP],localhost,[IP],[IPV6],socket-proxy,crowdsec\"\n213: ssh-keygen -q -t ed25519 -N \"\" -C \"homelab-crowdsec-capi-netbird-egress-edge\" -f \"$SSH_KEY\"\n217: echo \"EDGE_EGRESS_PUBKEY_FINGERPRINT\"\n221:install_egress() {\n223: echo \"INSTALL_EGRESS target=$TARGET_NB user=$TARGET_USER profile=$PROFILE bridge=$b\"\n225: echo \"VERIFY_NETBIRD_SSH_TO_TARGET\"\n234: \"echo SSH_OK; hostname; curl -4 -sk --connect-timeout 8 --max-time 20 -o /tmp/771u_capi_direct.body -w 'DIRECT_CAPI_HTTP=%{http_code}\\n' https://api.crowdsec.net/v3/watchers/login || true\" >\"$out\" 2>&1\n242: mkdir -p \"$EGRESS_DIR/profiles\"\n243: cat >\"$EGRESS_DIR/profiles/${PROFILE}.env\" </etc/systemd/system/homelab-crowdsec-capi-netbird-egress.service <<'EOF'\n255:Description=Homelab CrowdSec CAPI NetBird VPS SOCKS egress\n256:After=network-online.target docker.service netbird.service\nDEPENDENCY_FILE|path=/root/771v_deep_key_backup_netbird_capi.sh|mtime=2026-07-02 20:31:17.990220570 +0300|sha256=92191ce6124981ee0468cc8f95c749c47bca5fc61f9aecaeedeef91cdd434170\n8:PROOF=\"/root/evidence/771V_DEEP_KEY_BACKUP_NETBIRD_CAPI_${TS}_PROOF.txt\"\n11:EDGE_LOCAL=\"/root/771v_edge_netbird_capi_${TS}.sh\"\n12:EDGE_REMOTE=\"/tmp/771v_edge_netbird_capi_${TS}.sh\"\n29:STACK=/opt/stacks/crowdsec\n31:EGRESS_DIR=/etc/homelab-crowdsec-capi-netbird-egress\n32:HEALTH_DIR=/var/lib/homelab-health\n33:HEALTH_FILE=\"$HEALTH_DIR/crowdsec-capi.txt\"\n34:SSH_KEY=\"$EGRESS_DIR/id_ed25519\"\n37:mkdir -p \"$EGRESS_DIR/profiles\" \"$HEALTH_DIR\"\n38:chmod 700 \"$EGRESS_DIR\"\n45: docker network inspect crowdsec_default --format '{{(index .IPAM.Config 0).Gateway}}' 2>/dev/null || echo \"[IP]\"\n48:write_health() {\n52: echo \"CHECK=crowdsec-capi\"\n55: } > \"$HEALTH_FILE\"\n62: crowdsec:\n84: docker compose up -d crowdsec\n92: health=\"$(curl -sk --connect-timeout 3 --max-time 6 -o /tmp/771v_health.html -w '%{http_code}' http://[IP]:8088/health || true)\"\n95: fatal=\"$(docker logs --since 30s crowdsec 2>&1 | grep -Eic 'fatal|unable to run local API|failed to get docker info|lookup socket-proxy|server misbehaving|mapping value is not allowed' || true)\"\n96: echo \"LAPI_TRY=$n HEALTH=$health LAPI_RC=${rc:-NA} FATAL_COUNT_30S=$fatal\"\n98: if test \"$health\" = \"200\" && test \"${rc:-NA}\" = \"0\" && test \"$fatal\" = \"0\"; then\n133:remove(\"/opt/stacks/crowdsec/config/config.yaml\")\n134:remove(\"/opt/stacks/crowdsec/config/user.yaml\")\n141:p = Path(\"/opt/stacks/crowdsec/config/config.yaml\")\n172: \" credentials_path: /etc/crowdsec/online_api_credentials.yaml\",\n184: crowdsec:\n189: NO_PROXY: \"[IP],localhost,[IP],[IPV6],socket-proxy,crowdsec\"\n216: ssh-keygen -q -t ed25519 -N \"\" -C \"homelab-crowdsec-capi-netbird-egress-edge\" -f \"$SSH_KEY\"\n220: echo \"EDGE_EGRESS_PUBKEY_FINGERPRINT\"\n224:install_egress() {\n226: echo \"INSTALL_EGRESS target=$TARGET_NB user=$TARGET_USER profile=$PROFILE bridge=$b\"\n228: echo \"VERIFY_NETBIRD_SSH_TO_TARGET\"\n237: \"echo SSH_OK; hostname; curl -4 -sk --connect-timeout 8 --max-time 20 -o /tmp/771v_capi_direct.body -w 'DIRECT_CAPI_HTTP=%{http_code}\\n' https://api.crowdsec.net/v3/watchers/login || true\" >\"$out\" 2>&1\n245: mkdir -p \"$EGRESS_DIR/profiles\"\n246: cat >\"$EGRESS_DIR/profiles/${PROFILE}.env\" </etc/systemd/system/homelab-crowdsec-capi-netbird-egress.service <<'EOF'\nDEPENDENCY_FILE|path=/root/771z_after_manual_netbird_egress_capi.sh|mtime=2026-07-02 21:37:47.943243883 +0300|sha256=f198ed621fd726ed6f15c4a0dd49fad307befd01e7a58ebbda1ddc379f5b49ee\n8:PROOF=\"/root/evidence/771Z_AFTER_MANUAL_NETBIRD_EGRESS_CAPI_${TS}_PROOF.txt\"\n14: echo \"STEP=771Z_AFTER_MANUAL_NETBIRD_EGRESS_CAPI\"\n17: echo \"RULE=VERIFY_RELAY_MAIL_EGRESS_AND_REGISTER_CROWDSEC_CAPI\"\n18: echo \"TARGETS=edge-vm:[IP] relay-moldova:[IP] mail-usa:[IP]\"\n25:STACK=/opt/stacks/crowdsec\n27:HEALTH=/var/lib/homelab-health/crowdsec-capi.txt\n28:mkdir -p /var/lib/homelab-health\n34:write_health() {\n38: echo \"CHECK=crowdsec-capi\"\n41: } > \"$HEALTH\"\n49: health=\"$(curl -sk --connect-timeout 3 --max-time 6 -o /tmp/771z_health.html -w '%{http_code}' http://[IP]:8088/health || true)\"\n52: echo \"LAPI_TRY=$n HEALTH=$health LAPI_RC=${rc:-NA}\"\n54: test \"$health\" = \"200\" && test \"${rc:-NA}\" = \"0\" && return 0\n63: crowdsec:\n84: docker compose up -d crowdsec\n91: crowdsec:\n146:remove(\"/opt/stacks/crowdsec/config/config.yaml\")\n147:remove(\"/opt/stacks/crowdsec/config/user.yaml\")\n154:p = Path(\"/opt/stacks/crowdsec/config/config.yaml\")\n183: \" credentials_path: /etc/crowdsec/online_api_credentials.yaml\",\n194:echo \"NETBIRD_RESTART\"\n195:systemctl restart netbird 2>/dev/null || true\n198:echo \"NETBIRD_STATUS\"\n199:netbird status 2>&1 | redact || true\n201:echo \"PEERS_RELAY_MAIL_EDGE\"\n202:netbird status --json >/tmp/771z_nb.json 2>/tmp/771z_nb.err || true\n211: ip=peer.get(\"netbirdIp\",\"\")\n212: if ip in {\"[IP]\",\"[IP]\",\"[IP]\"} or any(x in fqdn for x in [\"relay\",\"mail\",\"edge-vm\"]):\n216:echo \"RELAY_MAIL_CONNECTIVITY\"\n224:echo \"WG_ALLOWED_IPS_RELAY_MAIL\"\n226: wg show wt0 allowed-ips 2>/dev/null | sed -E 's/^[A-Za-z0-9+\\/=]{20,}/PEERKEY_REDACTED/' | grep -E '100\\.100\\.19\\.1|100\\.100\\.147\\.204|0\\.0\\.0\\.0/0|100\\.100\\.0\\.0/16' || echo RELAY_MAIL_OR_DEFAULT_NOT_IN_ALLOWED_IPS\n229:echo \"CAPI_DIRECT_TEST_AFTER_MANUAL_NETBIRD\"\n230:capi_code=\"$(curl -4 -sk --http1.1 --connect-timeout 12 --max-time 45 -o /tmp/771z_capi.body -w '%{http_code}' https://api.crowdsec.net/v3/watchers/login || true)\"\n240: write_health \"REVIEW_MANUAL_NETBIRD_EGRESS_NOT_READY\" \"After manual NetBird setup, edge still cannot reach api.crowdsec.net; CrowdSec remains LAPI-only no-capi\"\n241: echo \"EDGE_STATUS=REVIEW_MANUAL_NETBIRD_EGRESS_NOT_READY_CAPI_NOT_DONE\"\n248:mkdir -p \"manual-backups/771z-$TS\"\nDEPENDENCY_FILE|path=/srv/homelab-ops/kb/current/03_ACCESS_AND_SECRETS.md|mtime=2026-08-21 00:06:53.672037607 +0300|sha256=e750284217dc41b5809268a0bbc54ed0948bdd14680fe2f282f307640937496c\nDEPENDENCY_FILE|path=/usr/local/sbin/homelab-alerting-health|mtime=2026-07-06 17:21:33.211361096 +0300|sha256=2050bac8c2ca8f1116e6d4fabe476292253bf38c56a49456242606980b4eb797\n4:OUT=\"/var/lib/homelab-health/alerting.txt\"\n5:mkdir -p /var/lib/homelab-health\n20:KUMA_STATE=\"$(ssh -n -o BatchMode=yes -o ConnectTimeout=8 \"debian@$MON_HOST\" \"sudo docker inspect -f '{{.State.Status}} {{if .State.Health}}{{.State.Health.Status}}{{end}}' uptime-kuma\" 2>/dev/null || true)\"\n23:GOTIFY_STATE=\"$(ssh -n -o BatchMode=yes -o ConnectTimeout=8 \"debian@$MON_HOST\" \"sudo docker inspect -f '{{.State.Status}}' gotify\" 2>/dev/null || true)\"\n24:if echo \"$GOTIFY_STATE\" | grep -q \"running\"; then ok gotify_container; else bad gotify_container; fi\n26:MON_ROW=\"$(ssh -n -o BatchMode=yes -o ConnectTimeout=8 \"debian@$MON_HOST\" \"sudo sqlite3 '$KUMA_DB' \\\"SELECT m.id || '|' || m.active || '|' || m.type || '|' || m.url || '|' || m.keyword || '|' || COALESCE(n.name,'') || '|' || COALESCE(n.active,'') FROM monitor m LEFT JOIN monitor_notification mn ON mn.monitor_id=m.id LEFT JOIN notification n ON n.id=mn.notification_id WHERE m.name='Homelab Overall Health' LIMIT 1;\\\"\" 2>/dev/null || true)\"\n30:if [[ \"${NNAME:-}\" = \"Gotify\" && \"${NACTIVE:-}\" = \"1\" ]]; then ok gotify_link; else bad gotify_link; fi\n33:echo \"STATUS=$STATUS TS=$TS TYPE=alerting BAD=$BAD$DETAILS kuma_state=${KUMA_STATE// /_} gotify_state=${GOTIFY_STATE// /_} monitor=${MON_ROW:-missing}\" > \"$OUT\"\nDEPENDENCY_FILE|path=/usr/local/sbin/homelab-kuma-monitor-policy|mtime=2026-07-06 17:21:33.211264665 +0300|sha256=87e1f14bde3e5eb2d0512c949e1e32d7fe3214f38797559b6e3e1a9d855f059a\n4:OUT=\"/var/lib/homelab-health/kuma-monitor-policy.txt\"\n12:PUBLIC_URL=\"https://backup.gram1.ru/overall.txt\"\n14:mkdir -p /var/lib/homelab-health \"$ROOT\"\n26:SQL=\"SELECT id||'|'||name||'|'||type||'|'||url||'|'||COALESCE(keyword,'')||'|'||active FROM monitor WHERE name IN ('Homelab Overall Health','Homelab Overall Public Health','File Browser Backup','File Browser Offhost Backup','File Browser Restore Validation','NPMplus Kuma Config Backup','Paperless Restore') ORDER BY id;\"\n30:LINKSQL=\"SELECT m.name||'|'||COALESCE(n.name,'')||'|'||COALESCE(n.active,'') FROM monitor m LEFT JOIN monitor_notification mn ON mn.monitor_id=m.id LEFT JOIN notification n ON n.id=mn.notification_id WHERE m.name IN ('Homelab Overall Health','Homelab Overall Public Health') ORDER BY m.name,n.name;\"\n34:printf \"%s\\n\" \"$ROWS\" | grep -Fq \"Homelab Overall Health|keyword|$PRIVATE_URL|STATUS=OK|1\" && ok private_monitor || bad private_monitor\n35:printf \"%s\\n\" \"$ROWS\" | grep -Fq \"Homelab Overall Public Health|keyword|$PUBLIC_URL|STATUS=OK|1\" && ok public_monitor || bad public_monitor\n36:printf \"%s\\n\" \"$LINKS\" | grep -Fxq \"Homelab Overall Health|Gotify|1\" && ok private_gotify || bad private_gotify\n37:printf \"%s\\n\" \"$LINKS\" | grep -Fxq \"Homelab Overall Public Health|Gotify|1\" && ok public_gotify || bad public_gotify\n39:LEGACY_FOUND=\"$(printf \"%s\\n\" \"$ROWS\" | awk -F'|' '$2==\"File Browser Backup\" || $2==\"File Browser Offhost Backup\" || $2==\"File Browser Restore Validation\" || $2==\"NPMplus Kuma Config Backup\" || $2==\"Paperless Restore\" {c++} END{print c+0}')\"\n40:LEGACY_ACTIVE=\"$(printf \"%s\\n\" \"$ROWS\" | awk -F'|' '($2==\"File Browser Backup\" || $2==\"File Browser Offhost Backup\" || $2==\"File Browser Restore Validation\" || $2==\"NPMplus Kuma Config Backup\" || $2==\"Paperless Restore\") && $6==\"1\" {c++} END{print c+0}')\"\n43:KUMA_STATE=\"$(ssh -n -o BatchMode=yes -o ConnectTimeout=8 \"$KUMA_HOST\" \"sudo docker inspect -f '{{.State.Status}} {{if .State.Health}}{{.State.Health.Status}}{{end}}' uptime-kuma\" 2>/dev/null || true)\"\nDEPENDENCY_FILE|path=/usr/local/sbin/homelab-runbook-generate|mtime=2026-07-03 14:21:01.771069001 +0300|sha256=956056878b1636369f39f8d79a1c725ebde17ee319bbbb5d5b09f188d656b23e\n7:health = Path(\"/var/lib/homelab-health/runbooks.txt\")\n8:state = Path(\"/var/lib/homelab-backup/apps\")\n12:health.parent.mkdir(parents=True, exist_ok=True)\n32: f\"## Current verified backup state\\n\"\n39: f\"- local_staging_deleted_after_two_cloud_verify: {d.get('local_backup_deleted', 'na')}\\n\\n\"\n43: f\"jq . /var/lib/homelab-backup/apps/latest-{app}.json\\n\"\n44: f\"appbackupctl restore-check {app}\\n\"\n50: f\"homelab-backupctl audit\\n\"\n53: f\"homelab-overall-health\\n\"\n73:index += \"homelab-overall-health\\n\"\n74:index += \"homelab-backupctl audit\\n\"\n85: \"backupctl restore-check\\n\"\n86: \"appbackupctl restore-check safety-net-control-plane\\n\"\n91: \"cat /var/lib/homelab-health/overall.txt\\n\"\n92: \"cat /var/lib/homelab-health/alerting.txt\\n\"\n93: \"ssh -n debian@[IP] \\\"sudo docker ps --format '{{.Names}}|{{.Status}}' | egrep 'uptime-kuma|gotify'\\\"\\n\"\n96: \"If backup verification, dependency map, or alerting health is not OK, stop automation and escalate manually.\\n\"\n108:health.write_text(line + \"\\n\")\nDEPENDENCY_FILE|path=/usr/local/sbin/homelab-vps-identity-audit|mtime=2026-06-24 16:00:07.135818676 +0300|sha256=b709691c52ae48432b70e197eb4810b6c374a05911cee3b9fdba4ecc67c76c3a\n5:VPS_ALIAS=\"${VPS_ALIAS:-netbird.vpn.gram1.ru}\"\n6:H=\"/var/lib/homelab-health/vps-identity-audit.txt\"\n9:mkdir -p /var/lib/homelab-health\n23:backup=\"$(find /mnt/staging/netbird-vps-backups/snapshots -maxdepth 2 -type f -name 'netbird-vps-backup.tgz' 2>/dev/null | sort | tail -n1 || true)\"\n24:backup_hostkeys=NO\n25:backup_match=NO\n27:if [ -n \"$backup\" ]; then\n28: if tar -tzf \"$backup\" | grep -Eq 'ssh_host_.*_key\\.pub$'; then\n29: backup_hostkeys=YES\n31: tar -xOzf \"$backup\" \"$k\" >> \"$TMP/backup-hostkeys.pub\" 2>/dev/null || true\n32: done < <(tar -tzf \"$backup\" | grep -E 'ssh_host_.*_key\\.pub$')\n33: ssh-keygen -lf \"$TMP/backup-hostkeys.pub\" > \"$TMP/backup-fingerprints.txt\" 2>/dev/null || true\n34: if grep -Fxf \"$TMP/current-fingerprints.txt\" \"$TMP/backup-fingerprints.txt\" >/dev/null 2>&1; then\n35: backup_match=YES\n41:[ \"$backup_hostkeys\" = \"YES\" ] && trust=\"BLOCKED_HOSTKEY_MISMATCH\"\n42:[ \"$backup_match\" = \"YES\" ] && trust=\"VERIFIED_BACKUP_MATCH\"\n46: \"$status\" \"$TS\" \"$VPS_IP\" \"$VPS_ALIAS\" \"${alias_ip:-NONE}\" \"$trust\" \"$backup_hostkeys\" > \"$H\"\nDEPENDENCY_FILE|path=/var/lib/homelab-backup-matrix/compat/netbird_vps_backup--netbird-vps-trust.txt--44dfdc0d5d0a.txt|mtime=2026-07-18 01:52:29.872264457 +0300|sha256=c1f8ec2f8be329eba2bed981a6c281a2b5d932ae36e8440b07895ff7db93084a\n3:TYPE=backup-matrix-generated\n4:JOB_ID=netbird_vps_backup\n5:JOB_NAME=netbird-vps-backup\n7:TIMER=netbird-vps-backup.timer\n8:SERVICE=netbird-vps-backup.service\n11:SOURCE_KIND=health_file\n12:EVIDENCE_LEVEL=artifact_or_health\n15:SOURCE_HEALTH=/var/lib/homelab-backup-matrix/evidence/netbird_vps_backup/0-netbird-vps-trust.txt\n21:LEGACY_SOURCE_PATH=/var/lib/homelab-health/netbird-vps-trust.txt\n22:CANONICAL_JOB_ID=netbird_vps_backup\n23:CANONICAL_STATE=/var/lib/homelab-backup-matrix/state.json\nDEPENDENCY_FILE|path=/var/lib/homelab-backup-matrix/evidence/homelab_verified_backup_health/0-backup-registry-latest.txt|mtime=2026-07-24 06:46:41.644124478 +0300|sha256=606f956d0c95435b4547a71a7ed3c5d8d0461700437815d064e3a7363e19ef21\n22:app=gotify job=homelab-pve01-app-gotify-20260724T014943Z status=success size=64413 deleted=true proxy_hosts=na certs=na\n25:app=healthchecks job=homelab-pve01-app-healthchecks-20260724T025849Z status=success size=686 deleted=true proxy_hosts=na certs=na\n53:Sat 2026-07-25 [IPV6] MSK 20h Fri 2026-07-24 [IPV6] MSK 3h 33min ago homelab-emergency-backup.timer homelab-emergency-backup.service\n54:Sat 2026-07-25 [IPV6] MSK 21h Fri 2026-07-24 [IPV6] MSK 2h 58min ago homelab-appbackup-dockge-stacks.timer homelab-appbackup-dockge-stacks.service\n55:Sat 2026-07-25 [IPV6] MSK 21h Fri 2026-07-24 [IPV6] MSK 2h 29min ago homelab-appbackup-npmplus-edge.timer homelab-appbackup-npmplus-edge.service\n56:Sat 2026-07-25 [IPV6] MSK 21h Fri 2026-07-24 [IPV6] MSK 2h 0min ago homelab-appbackup-sqlite-batch.timer homelab-appbackup-sqlite-batch.service\n57:Sat 2026-07-25 [IPV6] MSK 22h Fri 2026-07-24 [IPV6] MSK 1h 28min ago homelab-appbackup-postgres-batch.timer homelab-appbackup-postgres-batch.service\n58:Sat 2026-07-25 [IPV6] MSK 22h Fri 2026-07-24 [IPV6] MSK 52min ago homelab-appbackup-mariadb-batch.timer homelab-appbackup-mariadb-batch.service\n63:/var/lib/homelab-backup/quarantine/20260703T093243Z\nDEPENDENCY_FILE|path=/var/lib/homelab-backup-matrix/evidence-map.json|mtime=2026-07-10 12:47:24.326276710 +0300|sha256=0735c335850b22ee4947e0940c6cfd1cc1525f340f102b7f5bcfef5069f165c4\n6: \"job\": \"filebrowser-backup\",\n7: \"source_path\": \"/var/lib/homelab-health/filebrowser-backup.txt\",\n8: \"evidence_path\": \"/var/lib/homelab-backup-matrix/evidence/filebrowser_backup/0-filebrowser-backup.txt\"\n12: \"source_path\": \"/var/lib/homelab-health/filebrowser-restore.txt\",\n13: \"evidence_path\": \"/var/lib/homelab-backup-matrix/evidence/filebrowser_restore_validate/0-filebrowser-restore.txt\"\n16: \"job\": \"homelab-appbackup-dockge-stacks\",\n17: \"source_path\": \"/var/lib/homelab-backup/apps/latest-dockge-stacks.json\",\n18: \"evidence_path\": \"/var/lib/homelab-backup-matrix/evidence/homelab_appbackup_dockge_stacks/0-latest-dockge-stacks.json\"\n21: \"job\": \"homelab-appbackup-npmplus-edge\",\n22: \"source_path\": \"/var/lib/homelab-backup/apps/latest-npmplus-edge.json\",\n23: \"evidence_path\": \"/var/lib/homelab-backup-matrix/evidence/homelab_appbackup_npmplus_edge/0-latest-npmplus-edge.json\"\n26: \"job\": \"homelab-authentik-backup\",\n27: \"source_path\": \"/var/lib/homelab-health/authentik-backup.txt\",\n28: \"evidence_path\": \"/var/lib/homelab-backup-matrix/evidence/homelab_authentik_backup/0-authentik-backup.txt\"\n31: \"job\": \"homelab-authentik-backup\",\n32: \"source_path\": \"/var/lib/homelab-health/authentik-offhost.txt\",\n33: \"evidence_path\": \"/var/lib/homelab-backup-matrix/evidence/homelab_authentik_backup/1-authentik-offhost.txt\"\n36: \"job\": \"homelab-authentik-backup\",\n37: \"source_path\": \"/var/lib/homelab-health/authentik-restore.txt\",\n38: \"evidence_path\": \"/var/lib/homelab-backup-matrix/evidence/homelab_authentik_backup/2-authentik-restore.txt\"\n41: \"job\": \"homelab-backup-coverage-matrix\",\n42: \"source_path\": \"/var/lib/homelab-health/backup-coverage-matrix.txt\",\n43: \"evidence_path\": \"/var/lib/homelab-backup-matrix/evidence/homelab_backup_coverage_matrix/0-backup-coverage-matrix.txt\"\n46: \"job\": \"homelab-backup-sla-health\",\n47: \"source_path\": \"/var/lib/homelab-health/backup-sla.txt\",\n48: \"evidence_path\": \"/var/lib/homelab-backup-matrix/evidence/homelab_backup_sla_health/0-backup-sla.txt\"\n51: \"job\": \"homelab-edge-vm-vzdump-backup\",\n52: \"source_path\": \"/var/lib/homelab-health/edge-vm-vzdump-backup.txt\",\n53: \"evidence_path\": \"/var/lib/homelab-backup-matrix/evidence/homelab_edge_vm_vzdump_backup/0-edge-vm-vzdump-backup.txt\"\n56: \"job\": \"homelab-emergency-backup\",\n57: \"source_path\": \"/var/lib/homelab-backup/latest-emergency-pack.json\",\n58: \"evidence_path\": \"/var/lib/homelab-backup-matrix/evidence/homelab_emergency_backup/0-latest-emergency-pack.json\"\n61: \"job\": \"homelab-extended-appbackup\",\n62: \"source_path\": \"/var/lib/homelab-health/extended-appbackup.txt\",\n63: \"evidence_path\": \"/var/lib/homelab-backup-matrix/evidence/homelab_extended_appbackup/0-extended-appbackup.txt\"\n66: \"job\": \"homelab-gitea-backup\",\nDEPENDENCY_FILE|path=/var/lib/homelab-backup-matrix/evidence/netbird_vps_backup/0-netbird-vps-backup.txt|mtime=2026-07-01 04:31:05.433646905 +0300|sha256=372804695d6dba94e29e254f2175701e976692e7b15ee83fdcb6f10e05ca624e\n2:SERVICE=netbird-vps-backup\n5:SNAPSHOT=/mnt/staging/netbird-vps-backups/snapshots/20260701T013054Z\n6:LATEST_SYMLINK=/mnt/staging/netbird-vps-backups/snapshots/latest\nDEPENDENCY_FILE|path=/var/lib/homelab-backup-matrix/migration-plan.json|mtime=2026-07-10 13:10:05.080842534 +0300|sha256=260b6ef9d8c3370b6b429694ff642319ece1592f9e7c261e37b9fc3de0d33581\n3: \"dpkg-db-backup\",\n4: \"filebrowser-backup\",\n6: \"homelab-appbackup-dockge-stacks\",\n7: \"homelab-appbackup-mariadb-batch\",\n8: \"homelab-appbackup-npmplus-edge\",\n9: \"homelab-appbackup-postgres-batch\",\n10: \"homelab-appbackup-sqlite-batch\",\n11: \"homelab-auto-backup\",\n12: \"homelab-backup-coverage-matrix\",\n13: \"homelab-backup-freshness\",\n14: \"homelab-backup-healthcheck\",\n15: \"homelab-backup-sla-health\",\n16: \"homelab-edge-vm-vzdump-backup\",\n17: \"homelab-emergency-backup\",\n18: \"homelab-extended-appbackup\",\n19: \"homelab-mail-cloud-restore-drill\",\n20: \"homelab-verified-backup-health\",\n21: \"homelab-vm-backup-health\",\n22: \"homelab-vm-backup-policy\",\n23: \"homelab-vm-full-backup\",\n27: \"immich-media-backup\",\n28: \"memos-backup\",\n29: \"netbird-vps-backup\",\n30: \"paperless-backup\",\n32: \"skladchik-reports-monitor-backup-restore-check\",\n33: \"skladchik-reports-monitor-backup\"\n38: \"homelab-appbackup-postgres-batch\"\n40: \"owner_job\": \"homelab-appbackup-postgres-batch\",\n41: \"path\": \"/var/lib/homelab-health/authentik-backup.txt\"\n45: \"homelab-appbackup-postgres-batch\"\n47: \"owner_job\": \"homelab-appbackup-postgres-batch\",\n48: \"path\": \"/var/lib/homelab-health/authentik-offhost.txt\"\n52: \"homelab-appbackup-postgres-batch\"\n54: \"owner_job\": \"homelab-appbackup-postgres-batch\",\n55: \"path\": \"/var/lib/homelab-health/authentik-restore.txt\"\n59: \"homelab-backup-coverage-matrix\"\nDEPENDENCY_FILE|path=/var/lib/homelab-backup-matrix/rendered/netbird-vps-backup.txt|mtime=2026-08-30 21:08:09.037130131 +0300|sha256=d2d09878e7ccd4159c2c66225fea6bbcf30d7ba9a7e70ab30f5df4fae631620c\n3:TYPE=backup-matrix-generated\n4:JOB_ID=netbird_vps_backup\n5:JOB_NAME=netbird-vps-backup\n7:TIMER=netbird-vps-backup.timer\n8:SERVICE=netbird-vps-backup.service\n11:SOURCE_KIND=health_file\n12:EVIDENCE_LEVEL=artifact_or_health\n15:SOURCE_HEALTH=/var/lib/homelab-backup-matrix/evidence/netbird_vps_backup/0-netbird-vps-backup.txt\nDEPENDENCY_FILE|path=/var/lib/homelab-backup-matrix/state.json|mtime=2026-08-30 21:08:09.112843466 +0300|sha256=bba7caf3207a046105369da4d140f3342dec2e0897d91d5b4e2e099f5aab6b90\n3: \"compatibility_map\": \"/var/lib/homelab-backup-matrix/compat-map.json\",\n12: \"generated_health_output\": \"/var/lib/homelab-backup-matrix/rendered/dpkg-db-backup.txt\",\n13: \"id\": \"dpkg_db_backup\",\n15: \"name\": \"dpkg-db-backup\",\n16: \"service\": \"dpkg-db-backup.service\",\n19: \"source_health\": \"systemd:dpkg-db-backup.service\",\n24: \"timer\": \"dpkg-db-backup.timer\",\n32: \"evidence_level\": \"artifact_or_health\",\n34: \"generated_health_output\": \"/var/lib/homelab-backup-matrix/rendered/filebrowser-backup.txt\",\n35: \"id\": \"filebrowser_backup\",\n37: \"name\": \"filebrowser-backup\",\n38: \"service\": \"filebrowser-backup.service\",\n41: \"source_health\": \"/var/lib/homelab-backup/apps/latest-filebrowser.json\",\n42: \"source_kind\": \"health_file\",\n46: \"timer\": \"filebrowser-backup.timer\",\n54: \"evidence_level\": \"artifact_or_health\",\n56: \"generated_health_output\": \"/var/lib/homelab-backup-matrix/rendered/filebrowser-restore-validate.txt\",\n63: \"source_health\": \"/var/lib/homelab-backup-matrix/evidence/filebrowser_restore_validate/0-filebrowser-restore.txt\",\n64: \"source_kind\": \"health_file\",\n76: \"evidence_level\": \"artifact_or_health\",\n78: \"generated_health_output\": \"/var/lib/homelab-backup-matrix/rendered/homelab-appbackup-dockge-stacks.txt\",\n79: \"id\": \"homelab_appbackup_dockge_stacks\",\n81: \"name\": \"homelab-appbackup-dockge-stacks\",\n82: \"service\": \"homelab-appbackup-dockge-stacks.service\",\n85: \"source_health\": \"/var/lib/homelab-backup-matrix/evidence/homelab_appbackup_dockge_stacks/0-latest-dockge-stacks.json\",\n86: \"source_kind\": \"health_file\",\n90: \"timer\": \"homelab-appbackup-dockge-stacks.timer\",\n100: \"generated_health_output\": \"/var/lib/homelab-backup-matrix/rendered/homelab-appbackup-mariadb-batch.txt\",\n101: \"id\": \"homelab_appbackup_mariadb_batch\",\n103: \"name\": \"homelab-appbackup-mariadb-batch\",\n104: \"service\": \"homelab-appbackup-mariadb-batch.service\",\n107: \"source_health\": \"systemd:homelab-appbackup-mariadb-batch.service\",\n112: \"timer\": \"homelab-appbackup-mariadb-batch.timer\",\n120: \"evidence_level\": \"artifact_or_health\",\n122: \"generated_health_output\": \"/var/lib/homelab-backup-matrix/rendered/homelab-appbackup-npmplus-edge.txt\",\n123: \"id\": \"homelab_appbackup_npmplus_edge\",\nDEPENDENCY_FILE_INDEX_END=1\nRELAY_HEALTHCHECK_ARTIFACT_BEGIN=1\nARTIFACT_MISSING=/usr/local/sbin/pvepro-relay-healthcheck\nARTIFACT_MISSING=/etc/systemd/system/pvepro-relay-healthcheck.service\nARTIFACT_MISSING=/etc/systemd/system/pvepro-relay-healthcheck.timer\nARTIFACT_MISSING=/etc/systemd/system/pvepro-relay-healthcheck.service.d/20-gotify-notifier.conf\nRELAY_HEALTHCHECK_ARTIFACT_END=1\nNETBIRD_STATE_BEGIN=1\nNETBIRD_STATUS_JSON_RC=0\nNETBIRD_JSON_PARSE=true\nNETBIRD_DAEMON_STATUS=Connected\nNETBIRD_MANAGEMENT_CONNECTED=false\nNETBIRD_SIGNAL_CONNECTED=false\nNETBIRD_RELAYS_TOTAL=2\nNETBIRD_RELAYS_AVAILABLE=0\nNETBIRD_PEER|fqdn=pve02.netbird.selfhosted|status=Connected|connection=P2P\nNETBIRD_PEER|fqdn=home-pc.netbird.selfhosted|status=Connecting|connection=-\nNETBIRD_PEER|fqdn=relay.netbird.selfhosted|status=Connecting|connection=-\nNETBIRD_PEER|fqdn=pve03.netbird.selfhosted|status=Connected|connection=P2P\nNETBIRD_PEER|fqdn=edge-vm.netbird.selfhosted|status=Connecting|connection=-\nNETBIRD_PEER|fqdn=desktop-moy.netbird.selfhosted|status=Connecting|connection=-\nNETBIRD_PEER|fqdn=comp01.netbird.selfhosted|status=Connecting|connection=-\nNETBIRD_PEER|fqdn=e3qxxx.netbird.selfhosted|status=Connecting|connection=-\nNETBIRD_PEER|fqdn=mail.netbird.selfhosted|status=Connecting|connection=-\nNETBIRD_PEER|fqdn=desktop-server.netbird.selfhosted|status=Connecting|connection=-\nNETBIRD_PEER|fqdn=e3qxxx-183-106.netbird.selfhosted|status=Connecting|connection=-\nNETBIRD_NETWORKS_LIST_RC=0\nNo networks available.\nNETBIRD_ROUTES_LIST_RC=0\nNo networks available.\nNETBIRD_STATE_END=1\nPROCESS_AND_SOCKET_REFERENCES_BEGIN=1\n 1220 netbird /usr/bin/netbird service run --log-level info --daemon-addr unix:///var/run/netbird.sock --log-file /var/log/netbird/client.log\nESTAB 0 1500 [IP]:33490 [IP]:443 users:((\"netbird\",pid=1220,fd=35)) \nESTAB 0 1500 [IP]:33632 [IP]:443 users:((\"netbird\",pid=1220,fd=18)) \nESTAB 0 1500 [IP]:34548 [IP]:443 users:((\"netbird\",pid=1220,fd=43)) \nESTAB 0 1500 [IP]:37588 [IP]:443 users:((\"netbird\",pid=1220,fd=19)) \nESTAB 0 1500 [IP]:39034 [IP]:443 users:((\"netbird\",pid=1220,fd=20)) \nESTAB 0 1500 [IP]:39134 [IP]:443 users:((\"netbird\",pid=1220,fd=40)) \nESTAB 0 1500 [IP]:40402 [IP]:443 users:((\"netbird\",pid=1220,fd=11)) \nESTAB 0 1500 [IP]:41518 [IP]:443 users:((\"netbird\",pid=1220,fd=33)) \nESTAB 0 1500 [IP]:43330 [IP]:443 users:((\"netbird\",pid=1220,fd=17)) \nESTAB 0 1500 [IP]:43952 [IP]:443 users:((\"netbird\",pid=1220,fd=31)) \nESTAB 0 1500 [IP]:44564 [IP]:443 users:((\"netbird\",pid=1220,fd=41)) \nESTAB 0 1500 [IP]:47572 [IP]:443 users:((\"netbird\",pid=1220,fd=37)) \nESTAB 0 1500 [IP]:48242 [IP]:443 users:((\"netbird\",pid=1220,fd=30)) \nESTAB 0 0 [IP]:22 [IP]:61241 users:((\"sshd-session\",pid=2731736,fd=7),(\"sshd-session\",pid=2731520,fd=7))\nESTAB 0 1500 [IP]:50512 [IP]:443 users:((\"netbird\",pid=1220,fd=21)) \nESTAB 0 1500 [IP]:52460 [IP]:443 users:((\"netbird\",pid=1220,fd=16)) \nESTAB 0 1500 [IP]:53972 [IP]:443 users:((\"netbird\",pid=1220,fd=38)) \nESTAB 0 1500 [IP]:59556 [IP]:443 users:((\"netbird\",pid=1220,fd=39)) \nPROCESS_AND_SOCKET_REFERENCES_END=1\nDNS_AND_HOST_REFERENCES_BEGIN=1\nDNS_FILE=/etc/hosts|sha256=c3b9101ad96911d55328b70e12e87f9b17e4ef89fce47dc619478e113c4855ec\n11:[IP] pve01.gram1.ru pve01\n12:[IP] pve02.gram1.ru pve02\n13:[IP] pve03.gram1.ru pve03\nDNS_FILE=/etc/resolv.conf|sha256=be701c9795e17f831d600a2ad82e50a15785fb48a21d105d6413bb5b98a79fef\n1:# Generated by NetBird\n2:# The original file can be restored from /etc/resolv.conf.original.netbird\n4:search netbird.selfhosted gram1.ru\nDNS_AND_HOST_REFERENCES_END=1\nINTERPRETATION_RULES_BEGIN=1\nRULE_1=DO_NOT_DELETE_PROVIDER_VPS_WHILE_MD_RELAY_BACKUP_OR_MONITORING_REFERENCE_IS_ACTIVE_OR_UNRECONCILED\nRULE_2=DO_NOT_DELETE_WHILE_ANY_NETBIRD_ROUTE_OR_EXIT_NETWORK_STILL_SELECTS_MOLDOVA_FOR_AN_ACTIVE_PEER\nRULE_3=DO_NOT_DELETE_WHILE_PVEPRO_RELAY_HEALTHCHECK_OR_ALERTING_EXPECTS_THIS_NODE_UNLESS_REPLACED_OR_RETIRED\nRULE_4=DO_NOT_DELETE_WHILE_BACKUP_AUDIT_IS_FAILED_UNLESS_FAILURE_IS_PROVEN_UNRELATED_AND_BASELINE_IS_RESTORED\nRULE_5=AFTER_DETACH_USE_A_REVERSIBLE_QUARANTINE_WINDOW_BEFORE_PROVIDER_DELETION\nINTERPRETATION_RULES_END=1\nDECISION=HOLD_DELETE_PENDING_EXACT_PREFLIGHT_REVIEW\nNEXT_GATE=READ_EXACT_1130R1_AND_BUILD_NARROW_DETACH_OR_QUARANTINE_PLAN\nTASK_RESULT=PASS_MOLDOVA_RETIRE_PREFLIGHT_EVIDENCE_COLLECTED\nWORKERS2_MOLDOVA_RETIRE_PREFLIGHT_END=1\n" + "output": "CID=FORUM-260923-RCA-PVE02-VM160-AUDITC7A1R1\nMODE=read-only\nCOMPONENT=forum-pve02-vm160-audit\nMUTATION_BOUNDARY=NONE;PVE02_VM160_READ_ONLY;NO_MAIL_SEND;NO_DB_WRITE;NO_SERVICE_RESTART;NO_VM_CONFIG;NO_CLOUD\nEXECUTION_STARTED=true\nCHANGES_MADE=false\nROLLBACK_STARTED=false\nROLLBACK_RESTORED=null\nHANDOFF_SHA256=82a6060ccc216849c55790a3489ef157d5551a0c7cef0c5ec9abd35f569c7940\nPRIOR_CID=FORUM-260923-RCA-DEEP-AUDIT8D42R1\nPRIOR_FAILURE=VM160_NOT_RUNNING_ON_PVE01_WRONG_NODE\nCORRECTION=USE_PVE02_OWNER_NODE\nPVE02_TRANSPORT=PASS\nPVE02_EXPLICIT_KEY_USED=false\nPVE02_REMOTE_BEGIN=true\nPVE02_HOST=pve02\nVM160_STATE=running\nVM160_QGA_PING=OK\nVM160_QGA_TRANSPORT_RC=0\nVM160_GUEST_EXITCODE=0\nVM160_GUEST_AUDIT_BEGIN=true\nHOSTNAME=forum-prod\nTIME_UTC=2026-09-23T21:02:35Z\n=== SERVICES ===\nUNIT=nginx ACTIVE=active ENABLED=enabled\nUNIT=mariadb ACTIVE=active ENABLED=enabled\nUNIT=mysql ACTIVE=active ENABLED=alias\nUNIT=php8.3-fpm ACTIVE=active ENABLED=enabled\nUNIT=cron ACTIVE=active ENABLED=enabled\n=== FAILED UNITS ===\n=== CONFIG TESTS ===\nNGINX_CONFIGTEST_RC=0\nnginx: the configuration file /etc/nginx/nginx.conf syntax is ok\nnginx: configuration file /etc/nginx/nginx.conf test is successful\nPHPFPM_CONFIGTEST_BIN=php-fpm8.3 RC=0\n[23-Sep-2026 21:02:36] NOTICE: configuration file /etc/php/8.3/fpm/php-fpm.conf test is successful\n=== SENDMAIL/MSMTP STATE ===\nMAIL_BIN=/usr/sbin/sendmail mode=755 owner=root group=root size=143208\nMAIL_LINK=/usr/sbin/sendmail TARGET=../bin/msmtp\nMAIL_BIN=/usr/bin/msmtp mode=755 owner=root group=root size=143208\nMAIL_BIN_SHA256=d86ba2c0e12187ba16dcac322e4e94bd6079e635c1793b1690678b45bf3d0e61 PATH=/usr/bin/msmtp\nMAIL_BIN=/usr/local/sbin/codevipe-sendmail mode=755 owner=root group=root size=810\nMAIL_BIN_SHA256=e50ba5dcf4ac7a905ed1e6b1cc7c301a61318f9fe73976eeda3dd21047f39958 PATH=/usr/local/sbin/codevipe-sendmail\nMSMTP_CONFIG_META=/etc/msmtprc mode=640 owner=root group=xfp_mail size=319\nMSMTP_CONFIG_SHA256=c00fe15969542c6cda983e9b937488c9745e97e8fd8b2d75fdf6bd5386e722a9 PATH=/etc/msmtprc\nMSMTP_CONFIG_MISSING=/root/.msmtprc\nMSMTP_FILE_META=/etc/msmtp/pvepro.env mode=640 owner=root group=www-data size=150\nMSMTP_FILE_SHA256=32330a022cb6fbde19efb486c9945a6251070a544443a2947e7b222401a1ffee PATH=/etc/msmtp/pvepro.env\nsendmail_path => /usr/sbin/sendmail -t -i => /usr/sbin/sendmail -t -i\n/etc/php/8.3/fpm/pool.d/xf-dsmods.conf:2:php_admin_value[sendmail_path] = /usr/local/sbin/dsmods-sendmail -t -i\n/etc/php/8.3/fpm/pool.d/xf-kingofwolk.conf:2:php_admin_value[sendmail_path] = /usr/local/sbin/kingofwolk-sendmail -t -i\n/etc/php/8.3/fpm/pool.d/xf-codevipe.conf:2:php_admin_value[sendmail_path] = /usr/local/sbin/codevipe-sendmail -t -i\n/etc/php/8.3/fpm/pool.d/www.conf:486:;php_admin_value[sendmail_path] = /usr/sbin/sendmail -t -i -f \n/etc/php/8.3/fpm/pool.d/xf-zakrutim.conf:2:php_admin_value[sendmail_path] = /usr/local/sbin/zakrutim-sendmail -t -i\n/etc/php/8.3/fpm/pool.d/xf-hkmods.conf:2:php_admin_value[sendmail_path] = /usr/local/sbin/hkmods-sendmail -t -i\n/etc/php/8.3/fpm/pool.d/xf-gamevipe.conf:2:php_admin_value[sendmail_path] = /usr/local/sbin/gamevipe-sendmail -t -i\n=== MSMTP RECENT ERRORS ===\nMSMTP_LOG_PRESENT=/var/log/msmtp.log\nJul 01 07:19:46 host=mail.pvepro.ru tls=on auth=on user= from= recipients= smtpstatus=535 smtpmsg='535 5.7.8 Error: authentication failed: (reason unavailable)' errormsg='authentication failed (method PLAIN)' exitcode=EX_NOPERM\nJul 01 07:23:11 host=mail.pvepro.ru tls=on auth=on user= from= recipients= smtpstatus=535 smtpmsg='535 5.7.8 Error: authentication failed: (reason unavailable)' errormsg='authentication failed (method PLAIN)' exitcode=EX_NOPERM\nJul 01 07:23:19 host=mail.pvepro.ru tls=on auth=on user= from= recipients= smtpstatus=535 smtpmsg='535 5.7.8 Error: authentication failed: (reason unavailable)' errormsg='authentication failed (method PLAIN)' exitcode=EX_NOPERM\nJul 01 07:23:27 host=mail.pvepro.ru tls=on auth=on user= from= recipients= smtpstatus=535 smtpmsg='535 5.7.8 Error: authentication failed: (reason unavailable)' errormsg='authentication failed (method PLAIN)' exitcode=EX_NOPERM\nJul 01 08:08:27 host=mail.pvepro.ru tls=on auth=on user= from= recipients= mailsize=242 smtpstatus=250 smtpmsg='250 2.0.0 Ok: queued as 56D68A7F1C' exitcode=EX_OK\nJul 01 08:59:09 host=mail.pvepro.ru tls=on auth=on user= from= recipients= mailsize=1577 smtpstatus=250 smtpmsg='250 2.0.0 Ok: queued as 78667A73C3' exitcode=EX_OK\nJul 01 09:14:24 host=mail.pvepro.ru tls=on auth=on user= from= recipients= mailsize=1628 smtpstatus=250 smtpmsg='250 2.0.0 Ok: queued as D36DBA756C' exitcode=EX_OK\nJul 12 09:30:36 host=mail.pvepro.ru tls=on auth=on user= from= recipients= mailsize=7695 smtpstatus=250 smtpmsg='250 2.0.0 Ok: queued as 1166CA7FEF' exitcode=EX_OK\nJul 12 09:31:50 host=mail.pvepro.ru tls=on auth=on user= from= recipients= mailsize=7695 smtpstatus=250 smtpmsg='250 2.0.0 Ok: queued as 839D2A7FEF' exitcode=EX_OK\nJul 16 14:13:27 host=mail.pvepro.ru tls=on auth=on user= from= recipients= mailsize=7695 smtpstatus=250 smtpmsg='250 2.0.0 Ok: queued as 8E3D8A7FF9' exitcode=EX_OK\nJul 16 14:14:37 host=mail.pvepro.ru tls=on auth=on user= from= recipients= mailsize=7695 smtpstatus=250 smtpmsg='250 2.0.0 Ok: queued as 839E3A73F4' exitcode=EX_OK\nJul 16 14:15:19 host=mail.pvepro.ru tls=on auth=on user= from= recipients= mailsize=7695 smtpstatus=250 smtpmsg='250 2.0.0 Ok: queued as D1B4BA8012' exitcode=EX_OK\nJul 19 21:07:23 host=mail.pvepro.ru tls=on auth=on user= from= recipients= mailsize=7701 smtpstatus=250 smtpmsg='250 2.0.0 Ok: queued as A6FB3A8121' exitcode=EX_OK\nJul 22 11:29:09 host=mail.pvepro.ru tls=on auth=on user= from= recipients= mailsize=7707 smtpstatus=250 smtpmsg='250 2.0.0 Ok: queued as 77305A8015' exitcode=EX_OK\nJul 24 05:51:06 host=mail.pvepro.ru tls=on auth=on user= from= recipients= mailsize=7681 smtpstatus=250 smtpmsg='250 2.0.0 Ok: queued as ED00EA734F' exitcode=EX_OK\nJul 26 06:38:28 host=mail.pvepro.ru tls=on auth=on user= from= recipients= mailsize=7744 smtpstatus=250 smtpmsg='250 2.0.0 Ok: queued as 2DEA8A7F61' exitcode=EX_OK\nJul 28 19:04:01 host=mail.pvepro.ru tls=on auth=on user= from= recipients= mailsize=7697 smtpstatus=250 smtpmsg='250 2.0.0 Ok: queued as 5CE98A737C' exitcode=EX_OK\nJul 28 19:05:08 host=mail.pvepro.ru tls=on auth=on user= from= recipients= mailsize=7697 smtpstatus=250 smtpmsg='250 2.0.0 Ok: queued as E140AA737C' exitcode=EX_OK\nJul 29 08:48:58 host=mail.pvepro.ru tls=on auth=on user= from= recipients= mailsize=7669 smtpstatus=250 smtpmsg='250 2.0.0 Ok: queued as 227FBA7431' exitcode=EX_OK\nJul 29 20:33:47 host=mail.pvepro.ru tls=on auth=on user= from= recipients= mailsize=7685 smtpstatus=250 smtpmsg='250 2.0.0 Ok: queued as 9EBCFA7F0E' exitcode=EX_OK\nJul 30 16:48:00 host=mail.pvepro.ru tls=on auth=on user= from= recipients= mailsize=7687 smtpstatus=250 smtpmsg='250 2.0.0 Ok: queued as B1345A737C' exitcode=EX_OK\nAug 17 08:05:32 host=mail.pvepro.ru tls=on auth=on user= from= recipients= mailsize=7738 smtpstatus=250 smtpmsg='250 2.0.0 Ok: queued as 3EF03A7F8D' exitcode=EX_OK\nAug 17 08:07:52 host=mail.pvepro.ru tls=on auth=on user= from= recipients= mailsize=7738 smtpstatus=250 smtpmsg='250 2.0.0 Ok: queued as 6635BA73AA' exitcode=EX_OK\nAug 18 16:58:38 host=postbox.cloud.yandex.net tls=on auth=on user=ajeups2v3l2qfl9t1k9a from= recipients= mailsize=440 smtpstatus=250 smtpmsg='250 2.6.0 OK: queued as DKS89L8IPX05.3NYOOU5KRIBAX@ingress2-vla' exitcode=EX_OK\nAug 18 18:59:07 host=postbox.cloud.yandex.net tls=on auth=on user=ajeups2v3l2qfl9t1k9a from= recipients= mailsize=492 smtpstatus=250 smtpmsg='250 2.6.0 OK: queued as DKSATUF4C1XX.329CV6B95FH4R@ingress2-vla' exitcode=EX_OK\nAug 21 15:02:48 host=postbox.cloud.yandex.net tls=on auth=on user=ajeups2v3l2qfl9t1k9a from= recipients= mailsize=7672 smtpstatus=250 smtpmsg='250 2.6.0 OK: queued as DKUPOJOWNNPA.3CH5IJH2A8PW1@ingress2-klg' exitcode=EX_OK\nSep 02 19:39:11 host=postbox.cloud.yandex.net tls=on auth=on user=ajeups2v3l2qfl9t1k9a from= recipients= smtpstatus=550 smtpmsg='550 5.4.1 identity not found RequestID: 488b75b4-c264-42eb-b6fc-0cae6b175a7a' errormsg='envelope from address not accepted by the server' exitcode=EX_DATAERR\nSep 02 19:43:53 host=smtp.yandex.ru tls=on auth=on user= from= recipients= mailsize=257 smtpstatus=250 smtpmsg='250 2.0.0 Ok: queued on mail-nwsmtp-smtp-production-main-95.klg.yp-c.yandex.net 1788378233-qhqjKj2mO8c0-EjXDwBNJ' exitcode=EX_OK\nSep 02 19:44:53 host=smtp.yandex.ru tls=on auth=on user= from= recipients= mailsize=254 smtpstatus=250 smtpmsg='250 2.0.0 Ok: queued on mail-nwsmtp-smtp-production-main-71.sas.yp-c.yandex.net 1788378293-riqme1Oc6uQ0-JA3QSjr3' exitcode=EX_OK\nSep 02 21:33:45 host=smtp.yandex.ru tls=on auth=on user= from= recipients= mailsize=7477 smtpstatus=250 smtpmsg='250 2.0.0 Ok: queued on mail-nwsmtp-smtp-production-main-99.klg.yp-c.yandex.net 1788384825-iXsB0HPeHeA0-t67v0c58' exitcode=EX_OK\nSep 03 04:35:14 host=smtp.yandex.ru tls=on auth=on user= from= recipients= mailsize=2390 smtpstatus=250 smtpmsg='250 2.0.0 Ok: queued on mail-nwsmtp-smtp-production-main-85.iva.yp-c.yandex.net 1788410114-DZclo65g6Sw0-ym39kyd6' exitcode=EX_OK\nSep 03 04:44:35 host=smtp.yandex.ru tls=on auth=on user= from= recipients= mailsize=2412 smtpstatus=250 smtpmsg='250 2.0.0 Ok: queued on mail-nwsmtp-smtp-production-main-89.sas.yp-c.yandex.net 1788410675-ZicUTjSgH4Y0-ino0SiK0' exitcode=EX_OK\nSep 04 04:09:17 host=smtp.yandex.ru tls=on auth=on user= from= recipients= mailsize=7474 smtpstatus=250 smtpmsg='250 2.0.0 Ok: queued on mail-nwsmtp-smtp-production-main-88.klg.yp-c.yandex.net 1788494957-H9Anioed9mI0-YAPrVYsC' exitcode=EX_OK\nSep 04 04:09:44 host=smtp.yandex.ru tls=on auth=on user= from= recipients= mailsize=20404 smtpstatus=250 smtpmsg='250 2.0.0 Ok: queued on mail-nwsmtp-smtp-production-main-59.klg.yp-c.yandex.net 1788494984-h9AZQbdd08c0-OXrP5vgQ' exitcode=EX_OK\nSep 04 04:19:31 host=smtp.yandex.ru tls=on auth=on user= from= recipients= mailsize=7632 smtpstatus=250 smtpmsg='250 2.0.0 Ok: queued on mail-nwsmtp-smtp-production-main-98.sas.yp-c.yandex.net 1788495571-UJAjYwgdJCg0-xBGi7vHN' exitcode=EX_OK\nSep 04 06:00:34 host=smtp.yandex.ru tls=on auth=on user= from= recipients= mailsize=7703 smtpstatus=250 smtpmsg='250 2.0.0 Ok: queued on mail-nwsmtp-smtp-production-main-59.iva.yp-c.yandex.net 1788501634-X0CkCDGeJqM0-ewqs8N37' exitcode=EX_OK\nSep 08 08:25:56 host=smtp.yandex.ru tls=on auth=on user= from= recipients= mailsize=7704 smtpstatus=250 smtpmsg='250 2.0.0 Ok: queued on mail-nwsmtp-smtp-production-main-89.sas.yp-c.yandex.net 1788855956-tPIpFURgCSw0-0bD0dt06' exitcode=EX_OK\nSep 10 00:10:02 host=smtp.yandex.ru tls=on auth=on user= from= recipients= smtpstatus=553 smtpmsg='553 5.7.1 Sender address rejected: user not found. 1788999002-1ACLU4egAmI0-BvLQE2bE' errormsg='envelope from address not accepted by the server' exitcode=EX_DATAERR\nSep 10 00:10:03 host=smtp.yandex.ru tls=on auth=on user= from= recipients= mailsize=10460 smtpstatus=250 smtpmsg='250 2.0.0 Ok: queued on mail-nwsmtp-smtp-production-main-71.sas.yp-c.yandex.net 1788999003-3ACiEWkcAmI0-sgTWLJGA' exitcode=EX_OK\nSep 10 00:10:05 host=smtp.yandex.ru tls=on auth=on user= from= recipients= mailsize=10599 smtpstatus=250 smtpmsg='250 2.0.0 Ok: queued on mail-nwsmtp-smtp-production-main-60.sas.yp-c.yandex.net 1788999005-4ACnvojfQa60-IhiFG02U' exitcode=EX_OK\nSep 10 00:10:06 host=smtp.yandex.ru tls=on auth=on user= from= recipients= smtpstatus=553 smtpmsg='553 5.7.1 Sender address rejected: user not found. 1788999006-6ACgTJhfxuQ0-ykq2gT2m' errormsg='envelope from address not accepted by the server' exitcode=EX_DATAERR\nSep 10 00:10:07 host=smtp.yandex.ru tls=on auth=on user= from= recipients= smtpstatus=553 smtpmsg='553 5.7.1 Sender address rejected: user not found. 1788999007-7ACYnudg6mI0-wrzZ9ta2' errormsg='envelope from address not accepted by the server' exitcode=EX_DATAERR\nSep 10 00:10:08 host=smtp.yandex.ru tls=on auth=on user= from= recipients= mailsize=10224 smtpstatus=250 smtpmsg='250 2.0.0 Ok: queued on mail-nwsmtp-smtp-production-main-58.iva.yp-c.yandex.net 1788999008-8ACm4u3oe8c0-EAVVXjUV' exitcode=EX_OK\nSep 13 16:14:26 host=smtp.yandex.ru tls=on auth=on user= from= recipients= mailsize=7702 smtpstatus=250 smtpmsg='250 2.0.0 Ok: queued on mail-nwsmtp-smtp-production-main-67.vla.yp-c.yandex.net 1789316066-QEVwdXHe9Os0-PH6Ad0od' exitcode=EX_OK\nSep 19 00:10:02 host=smtp.yandex.ru tls=on auth=on user= from= recipients= smtpstatus=553 smtpmsg='553 5.7.1 Sender address rejected: user not found. 1789776602-2ALqagFfpGk0-vq5tlHNF' errormsg='envelope from address not accepted by the server' exitcode=EX_DATAERR\nSep 19 00:10:03 host=smtp.yandex.ru tls=on auth=on user= from= recipients= smtpstatus=553 smtpmsg='553 5.7.1 Sender address rejected: user not found. 1789776603-3ALZ9cFg74Y0-Rbt8yJgu' errormsg='envelope from address not accepted by the server' exitcode=EX_DATAERR\nSep 19 00:10:04 host=smtp.yandex.ru tls=on auth=on user= from= recipients= smtpstatus=553 smtpmsg='553 5.7.1 Sender address rejected: user not found. 1789776604-4AL5ABFdSSw0-XwXfZVbq' errormsg='envelope from address not accepted by the server' exitcode=EX_DATAERR\nSep 19 00:10:05 host=smtp.yandex.ru tls=on auth=on user= from= recipients= smtpstatus=553 smtpmsg='553 5.7.1 Sender address rejected: user not found. 1789776605-5ALSb5FdeKo0-ugaNiHmf' errormsg='envelope from address not accepted by the server' exitcode=EX_DATAERR\nSep 19 00:10:06 host=smtp.yandex.ru tls=on auth=on user= from= recipients= smtpstatus=553 smtpmsg='553 5.7.1 Sender address rejected: user not found. 1789776606-6ALAYdFdNuQ0-dL2d7DHS' errormsg='envelope from address not accepted by the server' exitcode=EX_DATAERR\nSep 19 00:10:07 host=smtp.yandex.ru tls=on auth=on user= from= recipients= smtpstatus=553 smtpmsg='553 5.7.1 Sender address rejected: user not found. 1789776607-7ALu7bFe4Gk0-WcjF4jNs' errormsg='envelope from address not accepted by the server' exitcode=EX_DATAERR\n=== JOB RUNNER ===\nJOBRUNNER_META=/root/scripts/forum-xenforo-run-jobs.sh mode=700 owner=root group=root size=2343\nJOBRUNNER_SHA256=5cc6c6cba317e0f0c4b1b13f375d8677b3c9c96d0416f998315a7d4e05e8a1aa PATH=/root/scripts/forum-xenforo-run-jobs.sh\nJOBRUNNER_META=/etc/cron.d/forum-xenforo-run-jobs mode=644 owner=root group=root size=183\nJOBRUNNER_SHA256=20cf9e4d059148c14094d145e778bb307b21183f608ff9b430dd620b9ece75c4 PATH=/etc/cron.d/forum-xenforo-run-jobs\n=== XENFORO SITES ===\nXENFORO_ROOT_COUNT=6\nSITE_BEGIN=1\nROOT_SHA256=90d10058f0574d8a5ff1ff412fb7b6a1eab2119d09c8b8162a0f8ca55d15180f\nDB_CONNECT=OK\nBOARD_DOMAIN=codevipe.ru\nEMAIL_TRANSPORT={\"emailTransport\":\"sendmail\"}\nDEFAULTEMAILADDRESS_PRESENT=true\nDEFAULTEMAILADDRESS_SHA256=2d4e7d3cf84ec1ba079400cf62ca99e7464a221a686ebe73405b73fb01ffe6f4\nBOUNCEEMAILADDRESS_PRESENT=true\nBOUNCEEMAILADDRESS_SHA256=1b5fc8a44aa74c2f02b7ce893f8a0e0a901d60b3f5e68b39c9448e7265117265\nCONTACTEMAILADDRESS_PRESENT=true\nCONTACTEMAILADDRESS_SHA256=0d4cf60ca8cd53a12ad50dc5419d3c9c32192f0852000eb10051fbdef52b04bf\nXF_ERROR_COUNT_24H=0\nXF_ERROR_COUNT_7D=2\nXF_ERROR_COUNT_30D=2\nXF_ERROR_COUNT_ALL=2\nXF_ERROR_TYPE=XF\\Db\\InvalidQueryException COUNT=1 LAST=1789960536\nXF_ERROR_TYPE=Symfony\\Component\\Mailer\\Exception\\TransportException COUNT=1 LAST=1789776602\nXF_ERROR_ROW ID=2 TS=1789960536 TYPE=XF\\Db\\InvalidQueryException FILE=AbstractStatement.php LINE=225 MSG=MySQL statement prepare error [1064]: You have an error in your SQL syntax; check the manual that corresponds to your MariaDB server version for the right syntax to use near 'FROM xf_template_modification WHERE addon_id='Gram/Support' ORDER MSG_SHA256=9ad3a33fd2ebdd213c22e2156d0358b02f9f6f82e2027f3101055bbfe5dd83b5\nXF_ERROR_ROW ID=1 TS=1789776602 TYPE=Symfony\\Component\\Mailer\\Exception\\TransportException FILE=ProcessStream.php LINE=74 MSG=Email to from failed: Process failed with exit code 65: sendmail: envelope from address not accepted by the server sendmail: server message: 553 5.7.1 Sender address rejected: user not found. 1789776602-2ALqagFfpGk0- MSG_SHA256=6ec17ccf28a8f55861f6ceb88ab722165bd922e90a12976f2755bc9db10c3638\nXF_MAIL_QUEUE_TABLE=MISSING\nXF_JOB_COUNT=2\nSITE_PHP_AUDIT_RC=0\nSITE_END=1\nSITE_BEGIN=2\nROOT_SHA256=d33b4c353c031284c61f9632f49971cd1f833ab4f48c0da27e4fdf81c373652e\nDB_CONNECT=OK\nBOARD_DOMAIN=dsmods.ru\nEMAIL_TRANSPORT={\"emailTransport\":\"sendmail\"}\nDEFAULTEMAILADDRESS_PRESENT=true\nDEFAULTEMAILADDRESS_SHA256=953017f372fc160381792c8f72822095c821e5f0c4519776c44cfa01fb6c16f1\nBOUNCEEMAILADDRESS_PRESENT=true\nBOUNCEEMAILADDRESS_SHA256=ba70f629ed27c307125a0538ca1c95049ac153aea2b21a474afeb4e3ad60d08d\nCONTACTEMAILADDRESS_PRESENT=true\nCONTACTEMAILADDRESS_SHA256=0d4cf60ca8cd53a12ad50dc5419d3c9c32192f0852000eb10051fbdef52b04bf\nXF_ERROR_COUNT_24H=0\nXF_ERROR_COUNT_7D=2\nXF_ERROR_COUNT_30D=2\nXF_ERROR_COUNT_ALL=2\nXF_ERROR_TYPE=XF\\Db\\InvalidQueryException COUNT=1 LAST=1789960537\nXF_ERROR_TYPE=Symfony\\Component\\Mailer\\Exception\\TransportException COUNT=1 LAST=1789776606\nXF_ERROR_ROW ID=2 TS=1789960537 TYPE=XF\\Db\\InvalidQueryException FILE=AbstractStatement.php LINE=225 MSG=MySQL statement prepare error [1064]: You have an error in your SQL syntax; check the manual that corresponds to your MariaDB server version for the right syntax to use near 'FROM xf_template_modification WHERE addon_id='Gram/Support' ORDER MSG_SHA256=9ad3a33fd2ebdd213c22e2156d0358b02f9f6f82e2027f3101055bbfe5dd83b5\nXF_ERROR_ROW ID=1 TS=1789776606 TYPE=Symfony\\Component\\Mailer\\Exception\\TransportException FILE=ProcessStream.php LINE=74 MSG=Email to from failed: Process failed with exit code 65: sendmail: envelope from address not accepted by the server sendmail: server message: 553 5.7.1 Sender address rejected: user not found. 1789776606-6ALAYdFdNuQ0- MSG_SHA256=59e1133715aedc5b530f2a2f87f31d659af13424d86fdc7b06d1331635178a0c\nXF_MAIL_QUEUE_TABLE=MISSING\nXF_JOB_COUNT=2\nSITE_PHP_AUDIT_RC=0\nSITE_END=2\nSITE_BEGIN=3\nROOT_SHA256=e06523a33d0b38a0ec6532783b5e63542eeba1303e68f2efad63214fd1e8908f\nDB_CONNECT=OK\nBOARD_DOMAIN=gamevipe.ru\nEMAIL_TRANSPORT={\"emailTransport\":\"sendmail\"}\nDEFAULTEMAILADDRESS_PRESENT=true\nDEFAULTEMAILADDRESS_SHA256=ded102c05ab32e41f2ada4d15b63c6adb848e14e76154015f294b751459a75dc\nBOUNCEEMAILADDRESS_PRESENT=true\nBOUNCEEMAILADDRESS_SHA256=3c6809f1cc3dd12044ad384b8d944a408bd84b4320af0bb505a5fb5ebf3f10e3\nCONTACTEMAILADDRESS_PRESENT=true\nCONTACTEMAILADDRESS_SHA256=0d4cf60ca8cd53a12ad50dc5419d3c9c32192f0852000eb10051fbdef52b04bf\nXF_ERROR_COUNT_24H=0\nXF_ERROR_COUNT_7D=1\nXF_ERROR_COUNT_30D=1\nXF_ERROR_COUNT_ALL=1\nXF_ERROR_TYPE=Symfony\\Component\\Mailer\\Exception\\TransportException COUNT=1 LAST=1789776603\nXF_ERROR_ROW ID=1 TS=1789776603 TYPE=Symfony\\Component\\Mailer\\Exception\\TransportException FILE=ProcessStream.php LINE=74 MSG=Email to from failed: Process failed with exit code 65: sendmail: envelope from address not accepted by the server sendmail: server message: 553 5.7.1 Sender address rejected: user not found. 1789776603-3ALZ9cFg74Y0- MSG_SHA256=f0aeb49dcba2c41db1f69663611a12e57913ca0b631287cdcc3c3b2a8db73f51\nXF_MAIL_QUEUE_TABLE=MISSING\nXF_JOB_COUNT=3\nSITE_PHP_AUDIT_RC=0\nSITE_END=3\nSITE_BEGIN=4\nROOT_SHA256=127c45667b0b6b6f23d20d6cc66d094b61a854801e42025ad2cb81ab5bfbc03e\nDB_CONNECT=OK\nBOARD_DOMAIN=hkmods.ru\nEMAIL_TRANSPORT={\"emailTransport\":\"sendmail\"}\nDEFAULTEMAILADDRESS_PRESENT=true\nDEFAULTEMAILADDRESS_SHA256=ef08c646d1d1fc56ef350fba44e29f2117996462bf46d9d14fa33c0688f9f4e3\nBOUNCEEMAILADDRESS_PRESENT=true\nBOUNCEEMAILADDRESS_SHA256=0097bdfb68ae6258a87c12c8dae4823421d4209086a2f190cb189defbda96bc3\nCONTACTEMAILADDRESS_PRESENT=true\nCONTACTEMAILADDRESS_SHA256=0d4cf60ca8cd53a12ad50dc5419d3c9c32192f0852000eb10051fbdef52b04bf\nXF_ERROR_COUNT_24H=0\nXF_ERROR_COUNT_7D=1\nXF_ERROR_COUNT_30D=1\nXF_ERROR_COUNT_ALL=1\nXF_ERROR_TYPE=Symfony\\Component\\Mailer\\Exception\\TransportException COUNT=1 LAST=1789776604\nXF_ERROR_ROW ID=1 TS=1789776604 TYPE=Symfony\\Component\\Mailer\\Exception\\TransportException FILE=ProcessStream.php LINE=74 MSG=Email to from failed: Process failed with exit code 65: sendmail: envelope from address not accepted by the server sendmail: server message: 553 5.7.1 Sender address rejected: user not found. 1789776604-4AL5ABFdSSw0- MSG_SHA256=03dcb6ecd92e29dde8dcccc9776bb4af1985d7d525bfae5cf9b5bb4db579de95\nXF_MAIL_QUEUE_TABLE=MISSING\nXF_JOB_COUNT=3\nSITE_PHP_AUDIT_RC=0\nSITE_END=4\nSITE_BEGIN=5\nROOT_SHA256=d249590aad8f87daf95d1fed623405edb307aa6c9b8123b2aaf89da88fc117ab\nDB_CONNECT=OK\nBOARD_DOMAIN=kingofwolk.ru\nEMAIL_TRANSPORT={\"emailTransport\":\"sendmail\"}\nDEFAULTEMAILADDRESS_PRESENT=true\nDEFAULTEMAILADDRESS_SHA256=e3b5d95ee979b3970d8ac7a07e503a6f5247963b701b1d1ad0b4aca12fc41d3a\nBOUNCEEMAILADDRESS_PRESENT=true\nBOUNCEEMAILADDRESS_SHA256=e49d0b3b9f084cf1223dd2b108019277e2b51caf2cf1a63b3ba3853765aefc65\nCONTACTEMAILADDRESS_PRESENT=true\nCONTACTEMAILADDRESS_SHA256=0d4cf60ca8cd53a12ad50dc5419d3c9c32192f0852000eb10051fbdef52b04bf\nXF_ERROR_COUNT_24H=0\nXF_ERROR_COUNT_7D=1\nXF_ERROR_COUNT_30D=1\nXF_ERROR_COUNT_ALL=1\nXF_ERROR_TYPE=Symfony\\Component\\Mailer\\Exception\\TransportException COUNT=1 LAST=1789776607\nXF_ERROR_ROW ID=1 TS=1789776607 TYPE=Symfony\\Component\\Mailer\\Exception\\TransportException FILE=ProcessStream.php LINE=74 MSG=Email to from failed: Process failed with exit code 65: sendmail: envelope from address not accepted by the server sendmail: server message: 553 5.7.1 Sender address rejected: user not found. 1789776607-7ALu7bFe4Gk0- MSG_SHA256=075f2b1963657d083c11e214ecc8cfd4e01d30f9ac1723bc440808730e6fb809\nXF_MAIL_QUEUE_TABLE=MISSING\nXF_JOB_COUNT=2\nSITE_PHP_AUDIT_RC=0\nSITE_END=5\nSITE_BEGIN=6\nROOT_SHA256=b5bc9e40848a038adb66a5656917858172c1b70a14ccbe680e9d8f2260dda046\nDB_CONNECT=OK\nBOARD_DOMAIN=zakrutim.ru\nEMAIL_TRANSPORT={\"emailTransport\":\"sendmail\"}\nDEFAULTEMAILADDRESS_PRESENT=true\nDEFAULTEMAILADDRESS_SHA256=b719f473d6ff99139fef2749e636437b76aaf482963e82dde4ac69a296a2eab9\nBOUNCEEMAILADDRESS_PRESENT=true\nBOUNCEEMAILADDRESS_SHA256=a93aa0f45435a0e2c159c501fbe3cc70deaff82401d29d87a972e633f91df83a\nCONTACTEMAILADDRESS_PRESENT=true\nCONTACTEMAILADDRESS_SHA256=0d4cf60ca8cd53a12ad50dc5419d3c9c32192f0852000eb10051fbdef52b04bf\nXF_ERROR_COUNT_24H=0\nXF_ERROR_COUNT_7D=1\nXF_ERROR_COUNT_30D=1\nXF_ERROR_COUNT_ALL=1\nXF_ERROR_TYPE=Symfony\\Component\\Mailer\\Exception\\TransportException COUNT=1 LAST=1789776605\nXF_ERROR_ROW ID=1 TS=1789776605 TYPE=Symfony\\Component\\Mailer\\Exception\\TransportException FILE=ProcessStream.php LINE=74 MSG=Email to from failed: Process failed with exit code 65: sendmail: envelope from address not accepted by the server sendmail: server message: 553 5.7.1 Sender address rejected: user not found. 1789776605-5ALSb5FdeKo0- MSG_SHA256=894c6c5636182853711e897fbcf34d5881646135b6dffd7360737376e789d384\nXF_MAIL_QUEUE_TABLE=MISSING\nXF_JOB_COUNT=2\nSITE_PHP_AUDIT_RC=0\nSITE_END=6\n=== RECENT SERVICE ERROR COUNTS ===\nJOURNAL_ERROR_COUNT_24H_nginx=0\nJOURNAL_ERROR_COUNT_24H_mariadb=0\nJOURNAL_ERROR_COUNT_24H_mysql=0\nJOURNAL_ERROR_COUNT_24H_php8_3_fpm=0\nVM160_GUEST_AUDIT_END=true\nVM160_GUEST_STDERR_PRESENT=false\nVM160_QGA_PARSE_RC=0\nPVE02_REMOTE_END=true\nFAILURE=NONE\nDECISION=PASS_PVE02_VM160_AUDIT\nNEXT_GATE=REVIEW_FOR_BOUNDED_REPAIR\nTASK_RESULT=READ_ONLY_PVE02_VM160_FORUM_AUDIT_COMPLETE\nHOMELAB_RESULT_CONTRACT={\"version\":1,\"command_id\":\"FORUM-260923-RCA-PVE02-VM160-AUDITC7A1R1\",\"status\":\"OK\",\"changes_made\":false,\"rollback_started\":false,\"rollback_restored\":null}\n" } diff --git a/runtime/latest.txt b/runtime/latest.txt index 62ab1c1a..cb4d5b49 100644 --- a/runtime/latest.txt +++ b/runtime/latest.txt @@ -1,15 +1,15 @@ CHAT_OUTPUT_BEGIN -COMMAND_ID=SUPPORT-260923-HOMELAB-MOLDOVA-RETIRE-PREFLIGHT-1130R1 +COMMAND_ID=FORUM-260923-RCA-PVE02-VM160-AUDITC7A1R1 STATUS=OK RC=0 HOST=pve01 MODE=read-only -COMPONENT=homelab-moldova-retire-preflight +COMPONENT=forum-pve02-vm160-audit REFERENCE_REGISTER_CHECK=OK REFERENCE_SHA256=5a3d8e5154c41cb582a4e0aca68090be1f0138918bf82131a948df326f9d8d66 ERROR_REGISTER_CHECK=OK ERROR_REGISTER_SHA256=3b09a553ec0f527ed3afeed4753f52a74ee3036045bbfb3c685e9f8af4ba7ba0 -COMMAND_SHA256=724447a90ca704066487f2c058f6cbf14e3086e7b399e385e5bca6df62d1cd70 +COMMAND_SHA256=d0d58f5b0d26ba9465d13d1ce6a174370a5532296fef88f757fd4fb1f2152a12 DUPLICATE_FAILED_COMMAND_BLOCKED=false EXECUTION_STARTED=true CHANGE_DECLARED=false @@ -24,2090 +24,268 @@ MUTATION_OUTCOME=NO_MUTATION SANITIZED=yes SECRETS_INCLUDED=no PRIVATE_ADDRESSES_INCLUDED=no -RAW_EVIDENCE_SHA256=f75ed52bc7c1eb7c5418acf9a6f3c7d372de8ddb2bf445b6eb507958c8172cf8 -SANITIZED_OUTPUT_SHA256=c0f54cefd55350849427898c5de7f25cde594a78aba7943a1b3599863593f6ac +RAW_EVIDENCE_SHA256=73f06c6c0cffab9eaddcd407da9d716944ca4ef348dd03eacfc4e7c8ceea1ea3 +SANITIZED_OUTPUT_SHA256=73f06c6c0cffab9eaddcd407da9d716944ca4ef348dd03eacfc4e7c8ceea1ea3 OUTPUT_BEGIN -WORKERS2_MOLDOVA_RETIRE_PREFLIGHT_BEGIN=1 -COMMAND_ID=SUPPORT-260923-HOMELAB-MOLDOVA-RETIRE-PREFLIGHT-1130R1 +CID=FORUM-260923-RCA-PVE02-VM160-AUDITC7A1R1 MODE=read-only -COMPONENT=homelab-moldova-retire-preflight -MUTATION_BOUNDARY=NONE;LOCAL_DEPENDENCY_DISCOVERY_ONLY;NO_PROVIDER_CALL;NO_TARGET_SSH;NO_NETWORK_MUTATION;NO_SYSTEMD_ACTION;NO_DNS_CHANGE;NO_BACKUP_MUTATION;NO_GIT_WRITE;NO_SECRET_VALUE_READOUT -PREFLIGHT_PURPOSE=PROVE_WHAT_MUST_BE_DETACHED_OR_REPLACED_BEFORE_MOLDOVA_VPS_CAN_BE_QUARANTINED_OR_DELETED -REFERENCE_SHA256=5a3d8e5154c41cb582a4e0aca68090be1f0138918bf82131a948df326f9d8d66 -ERROR_REGISTER_SHA256=3b09a553ec0f527ed3afeed4753f52a74ee3036045bbfb3c685e9f8af4ba7ba0 -RUNNER_SHA256=b248a4c32c9cc64e5747e7dce6c7fc0a23f5124a77c71ce72e27a81aceae9d2d -AUTHORITY_GATE=PASS -FRESH_CONTEXT_BEGIN=1 -NOW_UTC=2026-09-23T20:58:51Z -HOSTNAME=pve01 -FRESH_CONTEXT_END=1 -HANDOFF_CANDIDATES_BEGIN=1 -HANDOFF_CANDIDATE_COUNT=4 -HANDOFF_FILE=/srv/homelab-ops/kb/current/04_P0_NEXT.md -HANDOFF_MTIME=2026-08-21 00:06:53.672037607 +0300 -HANDOFF_SHA256=0f6eb8b23382becc1868e8a22318d5b7629184568205bad47c3c38967a346b8f -5:3. Canonical post-migration CR including NetBird new USA, Postbox/Null MX, Moldova V3/Gotify, old-USA retirement window and dependencies. -7:5. VM160 backup-state audit. -12:Do not mix unrelated changes into these gates (e.g. NetBird version upgrade/hostname change during SSH hardening). -HANDOFF_FILE=/srv/homelab-ops/kb/changes/CR-2026-0087-HANDOFF.md -HANDOFF_MTIME=2026-08-21 00:06:53.671922428 +0300 -HANDOFF_SHA256=a8d91b33b36e462bf3bb48defd12e121ed06edcb12d2dab3a9be50b2c5cdcb4d -13:- Scrutiny and crypto chart-health runtime are removed. -HANDOFF_FILE=/srv/homelab-ops/kb/changes/CR-2026-0086-HANDOFF.md -HANDOFF_MTIME=2026-08-21 00:06:53.671835127 +0300 -HANDOFF_SHA256=faab79cda5c6b31162b68812fc468359c0b9b94892c8f1e3b31c626d6704264a -26:3. load regression/error registry; -68:Failure: clean clone had no Git `user.name` / `user.email`. -138:V4 service was deployed side-by-side and later verified healthy, but public -199:- regression registry -HANDOFF_FILE=/srv/homelab-ops/kb/AI_CONTEXT.md -HANDOFF_MTIME=2026-08-21 00:06:53.671629955 +0300 -HANDOFF_SHA256=5fc43e34a1f0714ddacf7c00ed5c4bbaf788df3b8407dcfebd1333a23a623acc -15:- pve01 `[IP]`, operator historical NetBird path `root@[IP]`, MobaXterm key label `eddsa-key-20260617`. -26:## Mail/forums -30:## NetBird migration -32:Production NetBird is new USA `[IP]`, Debian 12, hostname `nb2`, NetBird `0.73.2`. Exact deployment caveat: `/api/health` = 404 and combined `:9000/health` = 503; these are not generic readiness gates for this deployment. Old USA `[IP]` has Mailcow/NetBird server stopped and observer disabled, but remains in rollback window. Permanent new-USA operator SSH hardening is still P0. -34:## Moldova -38:## Backups/cloud -40:Canonical production backup remotes remain Mail A/B. Full SITE-DR is SEALED; do not rerun merely because architecture changed. Cloud.ru bucket `homelab-dr-backup`: versioning **ON** (this supersedes older OFF docs), writer/reader OFF, no real workload. Yandex Object Storage bucket not created; latest verified billing threshold is `-600 RUB`, target 0, support request pending. Bucket creation remains blocked until live 0 proof. -HANDOFF_CANDIDATES_END=1 -REFERENCE_MOLDOVA_SNIPPETS_BEGIN=1 -4:SUPERSEDES_OLDER_TOPOLOGY_AND_BACKUP_SNAPSHOTS=YES -23:- VM180 cluster-admin | DESTROYED | backup registry lifecycle=RETIRED_PRESERVE -24:- CT200 skladchik-mod | DESTROYED | backup registry lifecycle=RETIRED_PRESERVE -25:- Scrutiny | RETIRED | active service/Podman removed | backup registry lifecycle=RETIRED_PRESERVE | historical recovery evidence preserved -28:### BACKUP_CURRENT -29:BACKUP_STATUS=GREEN -30:BACKUP_REGISTRY_TOTAL=21 -31:BACKUP_ACTIVE_VM=11 -32:BACKUP_RETIRED_VM=2 -33:BACKUP_ACTIVE_APP=4 -34:BACKUP_RETIRED_APP=3 -35:BACKUP_ACTIVE_CONTROL=1 -36:BACKUP_FRESH_TARGETS=15 -37:BACKUP_SOURCE_DRIFT=0 -38:BACKUP_RESUME=0 -39:BACKUP_PAYLOAD_LEAKS=0 -40:BACKUP_VM_QUEUE=inactive/success -41:BACKUP_APP_QUEUE=inactive/success -55:- Immich machine-learning runtime recovered; current stock healthcheck passes and server-to-ML /ping returns HTTP 200. -72:HEALTH_WARN_ERROR_COUNT=14 -74:NOTE forum-prod VMID160 is not in homelab-nightly-all; it has separate local/cloud backup chain -75:NOTE edge-vm disk scsi1 backup=0 risk must be documented -77:NOTE edge health WARN/ERROR items should be documented as known current states -79:HEALTH_NOT_OK -81:HEALTH_NOT_OK item=backup-restore-coverage-slo.txt line=STATUS=WARN -82:HEALTH_NOT_OK item=backup-restore-dashboard.txt line=STATUS=WARN TS=2026-06-29T18:10:09Z ITEMS=80 NOT_OK=13 -83:HEALTH_NOT_OK item=container-image-lifecycle-slo.txt line=STATUS=WARN -84:HEALTH_NOT_OK item=cron-job-monitoring-slo.txt line=STATUS=WARN -85:HEALTH_NOT_OK item=direct-heartbeat-pilot-readiness.txt line=STATUS=WARN -86:HEALTH_NOT_OK item=healthchecks-heartbeat-coverage.txt line=STATUS=WARN -87:HEALTH_NOT_OK item=healthchecks-job-coverage.txt line=STATUS=WARN -88:HEALTH_NOT_OK item=healthchecks-job-monitoring-slo.txt line=STATUS=WARN -89:HEALTH_NOT_OK item=homepage-service-catalog-slo.txt line=STATUS=WARN -90:HEALTH_NOT_OK item=ingress-dns-route-slo.txt line=STATUS=WARN -91:HEALTH_NOT_OK item=job-monitoring-slo.txt line=STATUS=WARN -92:HEALTH_NOT_OK item=observability-health-surface-slo.txt line=STATUS=WARN -REFERENCE_MOLDOVA_SNIPPETS_END=1 -SYSTEMD_STATE_BEGIN=1 -UNIT|name=pvepro-relay-healthcheck.timer|load=not-found|active=inactive|sub=dead|result=success|enabled=not-found -UNIT|name=pvepro-relay-healthcheck.service|load=not-found|active=inactive|sub=dead|result=success|enabled=not-found -UNIT|name=homelab-backup-v2.timer|load=loaded|active=active|sub=waiting|result=success|enabled=enabled -UNIT|name=homelab-backup-v2.service|load=loaded|active=inactive|sub=dead|result=success|enabled=static -UNIT|name=homelab-backup-audit.timer|load=loaded|active=active|sub=waiting|result=success|enabled=enabled -UNIT|name=homelab-backup-audit.service|load=loaded|active=failed|sub=failed|result=exit-code|enabled=static -UNIT|name=homelab-alertmanager-backup-health.timer|load=loaded|active=active|sub=waiting|result=success|enabled=enabled -UNIT|name=homelab-alertmanager-backup-health.service|load=loaded|active=inactive|sub=dead|result=success|enabled=static -UNIT|name=netbird-peers-health.timer|load=loaded|active=active|sub=waiting|result=success|enabled=enabled -UNIT|name=netbird-peers-health.service|load=loaded|active=failed|sub=failed|result=exit-code|enabled=static -UNIT|name=netbird.service|load=loaded|active=active|sub=running|result=success|enabled=enabled -UNIT|name=netbird-vps-backup.timer|load=not-found|active=inactive|sub=dead|result=success|enabled=not-found -UNIT|name=netbird-vps-backup.service|load=not-found|active=inactive|sub=dead|result=success|enabled=not-found -UNIT|name=netbird-vps-offhost-sync.timer|load=not-found|active=inactive|sub=dead|result=success|enabled=not-found -UNIT|name=netbird-vps-offhost-sync.service|load=not-found|active=inactive|sub=dead|result=success|enabled=not-found -UNIT|name=homelab-crowdsec-capi-netbird-egress.service|load=not-found|active=inactive|sub=dead|result=success|enabled=not-found -UNIT|name=privoxy.service|load=not-found|active=inactive|sub=dead|result=success|enabled=not-found -SYSTEMD_STATE_END=1 -BACKUP_AUDIT_FAILURE_EVIDENCE_BEGIN=1 -Sep 23 [IPV6] pve01 homelab-backup-audit[2268991]: TARGET|logical_id=app-filebrowser|kind=APP|state=FRESH|decision=SKIP_FRESH|legacy_age_seconds=14746|generation=app-filebrowser-pve01-20260923T143005Z -Sep 23 [IPV6] pve01 homelab-backup-audit[2268991]: TARGET|logical_id=app-hbarhub|kind=APP|state=FRESH|decision=SKIP_FRESH|legacy_age_seconds=14574|generation=app-hbarhub-pve01-20260923T143158Z -Sep 23 [IPV6] pve01 homelab-backup-audit[2268991]: TARGET|logical_id=app-immich|kind=APP|state=FRESH|decision=SKIP_FRESH|legacy_age_seconds=14674|generation=app-immich-pve01-20260923T143041Z -Sep 23 [IPV6] pve01 homelab-backup-audit[2268991]: TARGET|logical_id=app-skladchik-reports-monitor|kind=APP|state=FRESH|decision=SKIP_FRESH|legacy_age_seconds=14724|generation=app-skladchik-reports-monitor-pve01-20260923T143025Z -Sep 23 [IPV6] pve01 homelab-backup-audit[2268991]: TARGET|logical_id=control-bootstrap-v2|kind=CONTROL|state=FRESH|age_seconds=36576|generation=control-bootstrap-v2-pve01-20260916T120057Z -Sep 23 [IPV6] pve01 homelab-backup-audit[2268991]: TARGET|logical_id=vm110|kind=VM|state=FRESH|source=V2|v2_age_seconds=20203|legacy_age_seconds=1688189|replica_count=2|state_id=ct110|generation=v2-ct110-20260923T125546Z-34408ffe4182 -Sep 23 [IPV6] pve01 homelab-backup-audit[2268991]: TARGET|logical_id=vm111|kind=VM|state=FRESH|source=V2|v2_age_seconds=18061|replica_count=2|state_id=ct111|generation=v2-ct111-20260923T133222Z-0d3f5fae9c35 -Sep 23 [IPV6] pve01 homelab-backup-audit[2268991]: TARGET|logical_id=vm112|kind=VM|state=FRESH|source=V2|v2_age_seconds=71147|legacy_age_seconds=1726684|replica_count=2|state_id=ct112|generation=v2-ct112-20260922T224902Z-8457b570052f -Sep 23 [IPV6] pve01 homelab-backup-audit[2268991]: TARGET|logical_id=vm113|kind=VM|state=FRESH|source=V2|v2_age_seconds=69278|replica_count=2|state_id=ct113|generation=v2-ct113-20260922T231959Z-9fe7d084a330 -Sep 23 [IPV6] pve01 homelab-backup-audit[2268991]: TARGET|logical_id=vm130|kind=VM|state=FRESH|source=V2|v2_age_seconds=61252|replica_count=2|state_id=vm130|generation=v2-vm130-20260923T011558Z-c9498d37d3d7 -Sep 23 [IPV6] pve01 homelab-backup-audit[2268991]: TARGET|logical_id=vm150|kind=VM|state=FRESH|source=V2|v2_age_seconds=66934|legacy_age_seconds=1688305|replica_count=2|state_id=vm150|generation=v2-vm150-20260922T235842Z-71b8fcec59c9 -Sep 23 [IPV6] pve01 homelab-backup-audit[2268991]: TARGET|logical_id=vm160|kind=VM|state=FRESH|source=V2|v2_age_seconds=60152|replica_count=2|state_id=vm160|generation=v2-vm160-20260923T014608Z-f76d7671af5a -Sep 23 [IPV6] pve01 homelab-backup-audit[2268991]: TARGET|logical_id=vm170|kind=VM|state=FRESH|source=V2|v2_age_seconds=51801|legacy_age_seconds=632926|replica_count=2|state_id=vm170|generation=v2-vm170-20260923T035958Z-923adacbc94b -Sep 23 [IPV6] pve01 homelab-backup-audit[2268991]: TARGET|logical_id=vm171|kind=VM|state=FRESH|source=V2|v2_age_seconds=58355|legacy_age_seconds=662458|replica_count=2|state_id=vm171|generation=v2-vm171-20260923T021946Z-4d6e4f578075 -Sep 23 [IPV6] pve01 homelab-backup-audit[2268991]: TARGET|logical_id=vm190|kind=VM|state=FRESH|source=V2|v2_age_seconds=54272|replica_count=2|state_id=vm190|generation=v2-vm190-20260923T032759Z-314d9ba2ec35 -Sep 23 [IPV6] pve01 homelab-backup-audit[2268991]: TARGET|logical_id=vm9130|kind=VM|state=FRESH|source=V2|v2_age_seconds=39367|replica_count=2|state_id=vm9130|generation=v2-vm9130-20260923T071932Z-c0492db6535b -Sep 23 [IPV6] pve01 homelab-backup-audit[2268991]: ISSUE|severity=HIGH|code=LOCK_BUSY|logical_id=/run/homelab-backup-heavy.lock|root_cause=ACTIVE_OR_ORPHAN_LOCK -Sep 23 [IPV6] pve01 homelab-backup-audit[2268991]: SUMMARY|AUDIT_STATUS=OK|BACKUP_STATUS=RED|ACTIVE_COUNT=16|ISSUE_COUNT=1|CLOUD_A=OK|CLOUD_B=OK|CLOUD_B_DRILL=OK|DR_PROVEN=YES|SOURCE_OFFHOST=OK -Sep 23 [IPV6] pve01 systemd[1]: homelab-backup-audit.service: Main process exited, code=exited, status=1/FAILURE -Sep 23 [IPV6] pve01 systemd[1]: homelab-backup-audit.service: Failed with result 'exit-code'. -Sep 23 [IPV6] pve01 systemd[1]: Failed to start homelab-backup-audit.service - Homelab unified backup and DR audit. -Sep 23 [IPV6] pve01 systemd[1]: homelab-backup-audit.service: Consumed 2.367s CPU time, 148.3M memory peak. -Sep 23 [IPV6] pve01 systemd[1]: Starting homelab-backup-audit.service - Homelab unified backup and DR audit... -Sep 23 [IPV6] pve01 homelab-backup-audit[2372997]: TARGET|logical_id=app-filebrowser|kind=APP|state=FRESH|decision=SKIP_FRESH|legacy_age_seconds=16586|generation=app-filebrowser-pve01-20260923T143005Z -Sep 23 [IPV6] pve01 homelab-backup-audit[2372997]: TARGET|logical_id=app-hbarhub|kind=APP|state=FRESH|decision=SKIP_FRESH|legacy_age_seconds=16414|generation=app-hbarhub-pve01-20260923T143158Z -Sep 23 [IPV6] pve01 homelab-backup-audit[2372997]: TARGET|logical_id=app-immich|kind=APP|state=FRESH|decision=SKIP_FRESH|legacy_age_seconds=16514|generation=app-immich-pve01-20260923T143041Z -Sep 23 [IPV6] pve01 homelab-backup-audit[2372997]: TARGET|logical_id=app-skladchik-reports-monitor|kind=APP|state=FRESH|decision=SKIP_FRESH|legacy_age_seconds=16564|generation=app-skladchik-reports-monitor-pve01-20260923T143025Z -Sep 23 [IPV6] pve01 homelab-backup-audit[2372997]: TARGET|logical_id=control-bootstrap-v2|kind=CONTROL|state=FRESH|age_seconds=38415|generation=control-bootstrap-v2-pve01-20260916T120057Z -Sep 23 [IPV6] pve01 homelab-backup-audit[2372997]: TARGET|logical_id=vm110|kind=VM|state=FRESH|source=V2|v2_age_seconds=22042|legacy_age_seconds=1690029|replica_count=2|state_id=ct110|generation=v2-ct110-20260923T125546Z-34408ffe4182 -Sep 23 [IPV6] pve01 homelab-backup-audit[2372997]: TARGET|logical_id=vm111|kind=VM|state=FRESH|source=V2|v2_age_seconds=19900|replica_count=2|state_id=ct111|generation=v2-ct111-20260923T133222Z-0d3f5fae9c35 -Sep 23 [IPV6] pve01 homelab-backup-audit[2372997]: TARGET|logical_id=vm112|kind=VM|state=STALE|source=NONE|v2_age_seconds=72986|legacy_age_seconds=1728524|replica_count=2|state_id=ct112|generation=vm112-pve01-20260903T185707Z -Sep 23 [IPV6] pve01 homelab-backup-audit[2372997]: TARGET|logical_id=vm113|kind=VM|state=FRESH|source=V2|v2_age_seconds=71117|replica_count=2|state_id=ct113|generation=v2-ct113-20260922T231959Z-9fe7d084a330 -Sep 23 [IPV6] pve01 homelab-backup-audit[2372997]: TARGET|logical_id=vm130|kind=VM|state=FRESH|source=V2|v2_age_seconds=63091|replica_count=2|state_id=vm130|generation=v2-vm130-20260923T011558Z-c9498d37d3d7 -Sep 23 [IPV6] pve01 homelab-backup-audit[2372997]: TARGET|logical_id=vm150|kind=VM|state=FRESH|source=V2|v2_age_seconds=68773|legacy_age_seconds=1690145|replica_count=2|state_id=vm150|generation=v2-vm150-20260922T235842Z-71b8fcec59c9 -Sep 23 [IPV6] pve01 homelab-backup-audit[2372997]: TARGET|logical_id=vm160|kind=VM|state=FRESH|source=V2|v2_age_seconds=61991|replica_count=2|state_id=vm160|generation=v2-vm160-20260923T014608Z-f76d7671af5a -Sep 23 [IPV6] pve01 homelab-backup-audit[2372997]: TARGET|logical_id=vm170|kind=VM|state=FRESH|source=V2|v2_age_seconds=53640|legacy_age_seconds=634766|replica_count=2|state_id=vm170|generation=v2-vm170-20260923T035958Z-923adacbc94b -Sep 23 [IPV6] pve01 homelab-backup-audit[2372997]: TARGET|logical_id=vm171|kind=VM|state=FRESH|source=V2|v2_age_seconds=60194|legacy_age_seconds=664298|replica_count=2|state_id=vm171|generation=v2-vm171-20260923T021946Z-4d6e4f578075 -Sep 23 [IPV6] pve01 homelab-backup-audit[2372997]: TARGET|logical_id=vm190|kind=VM|state=FRESH|source=V2|v2_age_seconds=56111|replica_count=2|state_id=vm190|generation=v2-vm190-20260923T032759Z-314d9ba2ec35 -Sep 23 [IPV6] pve01 homelab-backup-audit[2372997]: TARGET|logical_id=vm9130|kind=VM|state=FRESH|source=V2|v2_age_seconds=41206|replica_count=2|state_id=vm9130|generation=v2-vm9130-20260923T071932Z-c0492db6535b -Sep 23 [IPV6] pve01 homelab-backup-audit[2372997]: ISSUE|severity=HIGH|code=RPO_BREACH|logical_id=vm112|root_cause=NO_FRESH_V2_OR_SEALED_QUORUM -Sep 23 [IPV6] pve01 homelab-backup-audit[2372997]: ISSUE|severity=HIGH|code=LOCK_BUSY|logical_id=/run/homelab-backup-heavy.lock|root_cause=ACTIVE_OR_ORPHAN_LOCK -Sep 23 [IPV6] pve01 homelab-backup-audit[2372997]: SUMMARY|AUDIT_STATUS=OK|BACKUP_STATUS=RED|ACTIVE_COUNT=16|ISSUE_COUNT=2|CLOUD_A=OK|CLOUD_B=OK|CLOUD_B_DRILL=OK|DR_PROVEN=YES|SOURCE_OFFHOST=OK -Sep 23 [IPV6] pve01 systemd[1]: homelab-backup-audit.service: Main process exited, code=exited, status=1/FAILURE -Sep 23 [IPV6] pve01 systemd[1]: homelab-backup-audit.service: Failed with result 'exit-code'. -Sep 23 [IPV6] pve01 systemd[1]: Failed to start homelab-backup-audit.service - Homelab unified backup and DR audit. -Sep 23 [IPV6] pve01 systemd[1]: homelab-backup-audit.service: Consumed 2.438s CPU time, 148.4M memory peak. -Sep 23 [IPV6] pve01 systemd[1]: Starting homelab-backup-audit.service - Homelab unified backup and DR audit... -Sep 23 [IPV6] pve01 homelab-backup-audit[2466119]: TARGET|logical_id=app-filebrowser|kind=APP|state=FRESH|decision=SKIP_FRESH|legacy_age_seconds=18320|generation=app-filebrowser-pve01-20260923T143005Z -Sep 23 [IPV6] pve01 homelab-backup-audit[2466119]: TARGET|logical_id=app-hbarhub|kind=APP|state=FRESH|decision=SKIP_FRESH|legacy_age_seconds=18148|generation=app-hbarhub-pve01-20260923T143158Z -Sep 23 [IPV6] pve01 homelab-backup-audit[2466119]: TARGET|logical_id=app-immich|kind=APP|state=FRESH|decision=SKIP_FRESH|legacy_age_seconds=18248|generation=app-immich-pve01-20260923T143041Z -Sep 23 [IPV6] pve01 homelab-backup-audit[2466119]: TARGET|logical_id=app-skladchik-reports-monitor|kind=APP|state=FRESH|decision=SKIP_FRESH|legacy_age_seconds=18298|generation=app-skladchik-reports-monitor-pve01-20260923T143025Z -Sep 23 [IPV6] pve01 homelab-backup-audit[2466119]: TARGET|logical_id=control-bootstrap-v2|kind=CONTROL|state=FRESH|age_seconds=40150|generation=control-bootstrap-v2-pve01-20260916T120057Z -Sep 23 [IPV6] pve01 homelab-backup-audit[2466119]: TARGET|logical_id=vm110|kind=VM|state=FRESH|source=V2|v2_age_seconds=23777|legacy_age_seconds=1691763|replica_count=2|state_id=ct110|generation=v2-ct110-20260923T125546Z-34408ffe4182 -Sep 23 [IPV6] pve01 homelab-backup-audit[2466119]: TARGET|logical_id=vm111|kind=VM|state=FRESH|source=V2|v2_age_seconds=21635|replica_count=2|state_id=ct111|generation=v2-ct111-20260923T133222Z-0d3f5fae9c35 -Sep 23 [IPV6] pve01 homelab-backup-audit[2466119]: TARGET|logical_id=vm112|kind=VM|state=STALE|source=NONE|v2_age_seconds=74721|legacy_age_seconds=1730258|replica_count=2|state_id=ct112|generation=vm112-pve01-20260903T185707Z -Sep 23 [IPV6] pve01 homelab-backup-audit[2466119]: TARGET|logical_id=vm113|kind=VM|state=STALE|source=NONE|v2_age_seconds=72852|replica_count=2|state_id=ct113 -Sep 23 [IPV6] pve01 homelab-backup-audit[2466119]: TARGET|logical_id=vm130|kind=VM|state=FRESH|source=V2|v2_age_seconds=64826|replica_count=2|state_id=vm130|generation=v2-vm130-20260923T011558Z-c9498d37d3d7 -Sep 23 [IPV6] pve01 homelab-backup-audit[2466119]: TARGET|logical_id=vm150|kind=VM|state=FRESH|source=V2|v2_age_seconds=70508|legacy_age_seconds=1691879|replica_count=2|state_id=vm150|generation=v2-vm150-20260922T235842Z-71b8fcec59c9 -Sep 23 [IPV6] pve01 homelab-backup-audit[2466119]: TARGET|logical_id=vm160|kind=VM|state=FRESH|source=V2|v2_age_seconds=63726|replica_count=2|state_id=vm160|generation=v2-vm160-20260923T014608Z-f76d7671af5a -Sep 23 [IPV6] pve01 homelab-backup-audit[2466119]: TARGET|logical_id=vm170|kind=VM|state=FRESH|source=V2|v2_age_seconds=55375|legacy_age_seconds=636500|replica_count=2|state_id=vm170|generation=v2-vm170-20260923T035958Z-923adacbc94b -Sep 23 [IPV6] pve01 homelab-backup-audit[2466119]: TARGET|logical_id=vm171|kind=VM|state=FRESH|source=V2|v2_age_seconds=61929|legacy_age_seconds=666032|replica_count=2|state_id=vm171|generation=v2-vm171-20260923T021946Z-4d6e4f578075 -Sep 23 [IPV6] pve01 homelab-backup-audit[2466119]: TARGET|logical_id=vm190|kind=VM|state=FRESH|source=V2|v2_age_seconds=57846|replica_count=2|state_id=vm190|generation=v2-vm190-20260923T032759Z-314d9ba2ec35 -Sep 23 [IPV6] pve01 homelab-backup-audit[2466119]: TARGET|logical_id=vm9130|kind=VM|state=FRESH|source=V2|v2_age_seconds=42941|replica_count=2|state_id=vm9130|generation=v2-vm9130-20260923T071932Z-c0492db6535b -Sep 23 [IPV6] pve01 homelab-backup-audit[2466119]: ISSUE|severity=HIGH|code=RPO_BREACH|logical_id=vm112|root_cause=NO_FRESH_V2_OR_SEALED_QUORUM -Sep 23 [IPV6] pve01 homelab-backup-audit[2466119]: ISSUE|severity=HIGH|code=RPO_BREACH|logical_id=vm113|root_cause=NO_FRESH_V2_OR_SEALED_QUORUM -Sep 23 [IPV6] pve01 homelab-backup-audit[2466119]: SUMMARY|AUDIT_STATUS=OK|BACKUP_STATUS=RED|ACTIVE_COUNT=16|ISSUE_COUNT=2|CLOUD_A=OK|CLOUD_B=OK|CLOUD_B_DRILL=OK|DR_PROVEN=YES|SOURCE_OFFHOST=OK -Sep 23 [IPV6] pve01 systemd[1]: homelab-backup-audit.service: Main process exited, code=exited, status=1/FAILURE -Sep 23 [IPV6] pve01 systemd[1]: homelab-backup-audit.service: Failed with result 'exit-code'. -Sep 23 [IPV6] pve01 systemd[1]: Failed to start homelab-backup-audit.service - Homelab unified backup and DR audit. -Sep 23 [IPV6] pve01 systemd[1]: homelab-backup-audit.service: Consumed 2.421s CPU time, 148M memory peak. -Sep 23 [IPV6] pve01 systemd[1]: Starting homelab-backup-audit.service - Homelab unified backup and DR audit... -Sep 23 [IPV6] pve01 homelab-backup-audit[2565382]: TARGET|logical_id=app-filebrowser|kind=APP|state=FRESH|decision=SKIP_FRESH|legacy_age_seconds=20177|generation=app-filebrowser-pve01-20260923T143005Z -Sep 23 [IPV6] pve01 homelab-backup-audit[2565382]: TARGET|logical_id=app-hbarhub|kind=APP|state=FRESH|decision=SKIP_FRESH|legacy_age_seconds=20005|generation=app-hbarhub-pve01-20260923T143158Z -Sep 23 [IPV6] pve01 homelab-backup-audit[2565382]: TARGET|logical_id=app-immich|kind=APP|state=FRESH|decision=SKIP_FRESH|legacy_age_seconds=20105|generation=app-immich-pve01-20260923T143041Z -Sep 23 [IPV6] pve01 homelab-backup-audit[2565382]: TARGET|logical_id=app-skladchik-reports-monitor|kind=APP|state=FRESH|decision=SKIP_FRESH|legacy_age_seconds=20155|generation=app-skladchik-reports-monitor-pve01-20260923T143025Z -Sep 23 [IPV6] pve01 homelab-backup-audit[2565382]: TARGET|logical_id=control-bootstrap-v2|kind=CONTROL|state=FRESH|age_seconds=42006|generation=control-bootstrap-v2-pve01-20260916T120057Z -Sep 23 [IPV6] pve01 homelab-backup-audit[2565382]: TARGET|logical_id=vm110|kind=VM|state=FRESH|source=V2|v2_age_seconds=25633|legacy_age_seconds=1693620|replica_count=2|state_id=ct110|generation=v2-ct110-20260923T125546Z-34408ffe4182 -Sep 23 [IPV6] pve01 homelab-backup-audit[2565382]: TARGET|logical_id=vm111|kind=VM|state=FRESH|source=V2|v2_age_seconds=23491|replica_count=2|state_id=ct111|generation=v2-ct111-20260923T133222Z-0d3f5fae9c35 -Sep 23 [IPV6] pve01 homelab-backup-audit[2565382]: TARGET|logical_id=vm112|kind=VM|state=STALE|source=NONE|v2_age_seconds=76577|legacy_age_seconds=1732115|replica_count=2|state_id=ct112|generation=vm112-pve01-20260903T185707Z -Sep 23 [IPV6] pve01 homelab-backup-audit[2565382]: TARGET|logical_id=vm113|kind=VM|state=STALE|source=NONE|v2_age_seconds=74708|replica_count=2|state_id=ct113 -Sep 23 [IPV6] pve01 homelab-backup-audit[2565382]: TARGET|logical_id=vm130|kind=VM|state=FRESH|source=V2|v2_age_seconds=66682|replica_count=2|state_id=vm130|generation=v2-vm130-20260923T011558Z-c9498d37d3d7 -Sep 23 [IPV6] pve01 homelab-backup-audit[2565382]: TARGET|logical_id=vm150|kind=VM|state=STALE|source=NONE|v2_age_seconds=72364|legacy_age_seconds=1693736|replica_count=2|state_id=vm150|generation=vm150-pve01-20260904T053517Z -Sep 23 [IPV6] pve01 homelab-backup-audit[2565382]: TARGET|logical_id=vm160|kind=VM|state=FRESH|source=V2|v2_age_seconds=65582|replica_count=2|state_id=vm160|generation=v2-vm160-20260923T014608Z-f76d7671af5a -Sep 23 [IPV6] pve01 homelab-backup-audit[2565382]: TARGET|logical_id=vm170|kind=VM|state=FRESH|source=V2|v2_age_seconds=57231|legacy_age_seconds=638357|replica_count=2|state_id=vm170|generation=v2-vm170-20260923T035958Z-923adacbc94b -Sep 23 [IPV6] pve01 homelab-backup-audit[2565382]: TARGET|logical_id=vm171|kind=VM|state=FRESH|source=V2|v2_age_seconds=63785|legacy_age_seconds=667889|replica_count=2|state_id=vm171|generation=v2-vm171-20260923T021946Z-4d6e4f578075 -Sep 23 [IPV6] pve01 homelab-backup-audit[2565382]: TARGET|logical_id=vm190|kind=VM|state=FRESH|source=V2|v2_age_seconds=59702|replica_count=2|state_id=vm190|generation=v2-vm190-20260923T032759Z-314d9ba2ec35 -Sep 23 [IPV6] pve01 homelab-backup-audit[2565382]: TARGET|logical_id=vm9130|kind=VM|state=FRESH|source=V2|v2_age_seconds=44797|replica_count=2|state_id=vm9130|generation=v2-vm9130-20260923T071932Z-c0492db6535b -Sep 23 [IPV6] pve01 homelab-backup-audit[2565382]: ISSUE|severity=HIGH|code=RPO_BREACH|logical_id=vm112|root_cause=NO_FRESH_V2_OR_SEALED_QUORUM -Sep 23 [IPV6] pve01 homelab-backup-audit[2565382]: ISSUE|severity=HIGH|code=RPO_BREACH|logical_id=vm113|root_cause=NO_FRESH_V2_OR_SEALED_QUORUM -Sep 23 [IPV6] pve01 homelab-backup-audit[2565382]: ISSUE|severity=HIGH|code=RPO_BREACH|logical_id=vm150|root_cause=NO_FRESH_V2_OR_SEALED_QUORUM -BACKUP_AUDIT_FAILURE_EVIDENCE_END=1 -HEALTH_FILES_BEGIN=1 -HEALTH_FILE=cluster-admin-vm-mail-cloud-backup.txt|mtime=2026-07-09 08:07:47.213452046 +0300|sha256=e03b236f41f8d66f070ecd3234b3ff0836a44d972fd6f6abc721b9683fa22053 -STATUS=OK TS=2026-07-09T05:07:47Z TYPE=cluster-admin-vm-mail-cloud-backup BAD=0 VMID=180 NODE=pve02 MODE=MAIL_CLOUD_2COPY_VERIFY_OK LOCAL_STATUS=OK REPORT=/var/lib/homelab-cluster-admin-vm-mail-cloud-backup/latest.md -HEALTH_FILE=alertmanager-backup.txt|mtime=2026-07-09 10:50:17.826712693 +0300|sha256=f5a8a0ad29606ebbdcdcb52a997e9ee07ddec9ab7eab6bb0932c3f4e14eba97a -STATUS=OK TS=2026-07-09T07:50:16Z TYPE=alertmanager-backup JOB=homelab-pve01-app-alertmanager-20260709T075016Z SOURCE=alertmanager_config SIZE=1842 SHA256=58fb1b3467a962acff44bf75cb274e3a17c06afdbd3a25314a904f75c8048728 LOCAL_BACKUP_DELETED=true NOTE=service-readiness-backup-proof -HEALTH_FILE=mealie-backup.txt|mtime=2026-07-09 19:17:35.017044807 +0300|sha256=8043626489c0b50dc30f11771e15940c0cb27c99cebdf482cf7dfba83571d074 -STATUS=OK TS=2026-07-09T16:17:31Z TYPE=mealie-backup JOB=homelab-pve01-app-mealie-20260709T051713Z SOURCE=latest-appbackup SIZE=912487 LOCAL_BACKUP_DELETED=true JSON=/var/lib/homelab-backup/apps/latest-mealie.json NOTE=service-readiness-backup-proof -HEALTH_FILE=karakeep-backup.txt|mtime=2026-07-09 22:28:35.572436986 +0300|sha256=79a25462c21ee9006462c6dc6fb7c2b21a4e7c297f585b770ed1e9a77e718230 -STATUS=OK TS=2026-07-09T19:28:34Z TYPE=karakeep-backup BAD=0 APP=karakeep JOB=homelab-pve01-app-karakeep-20260709T051713Z SIZE=23140 LOCAL_BACKUP_DELETED=true JSON=/var/lib/homelab-backup/apps/latest-karakeep.json -HEALTH_FILE=backup-restore-dashboard.txt|mtime=2026-07-03 00:45:30.941379161 +0300|sha256=51b95b7465dc040ff080c8a8341b23c9c0fdd90cfe0675ccd3f705b55590d815 -STATUS=OK -TS=20260702T214530Z -TYPE=backup-restore-dashboard-rebuilt -ITEMS=13 -NOT_OK=0 -NOT_OK_LIST=none -ITEM name=backup.txt exists=true ok=true -ITEM name=restore-drill-index.txt exists=true ok=true -ITEM name=paperless-backup.txt exists=true ok=true -ITEM name=paperless-offhost.txt exists=true ok=true -ITEM name=paperless-restore.txt exists=true ok=true -ITEM name=npmplus-kuma-config-backup.txt exists=true ok=true -ITEM name=npmplus-kuma-config-offhost.txt exists=true ok=true -ITEM name=npmplus-kuma-config-restore.txt exists=true ok=true -ITEM name=netbird-peers.txt exists=true ok=true -ITEM name=crowdsec-capi-netbird-final.txt exists=true ok=true -ITEM name=adguard-rewrite-sync.txt exists=true ok=true -ITEM name=router-running-config-mail-cloud.txt exists=true ok=true -ITEM name=mail-cloud-restore-drill.txt exists=true ok=true -HEALTH_FILE=backup-matrix-cluster-final-seal.txt|mtime=2026-07-13 20:51:32.422338540 +0300|sha256=dcd06c2ff207c2922fc68d4df4b42ffa4b6680785e532783772e764feb828aa1 -STATUS=OK TS=2026-07-13T17:51:31Z TYPE=backup-matrix-cluster-final-seal BAD=0 CURRENT_STAGE_COMPLETE=YES MATRIX_STATUS=OK JOBS_TOTAL=35 JOBS_OK=31 JOBS_BAD=0 JOBS_UNMAPPED=0 JOBS_DISABLED=4 READ_WRITE_OVERLAP=0 VM_BACKUP_HEALTH=OK VM_FULL_BACKUP=OK EDGE_VM_CANONICAL_BACKUP=homelab-edge-vm-offhost-freshness.timer OLD_EDGE_TIMER_REENABLED=NO CLUSTER_PASSPORT=OK FINAL_READINESS=OK FINAL_READINESS_PASS=53 FINAL_READINESS_FAIL=0 OVERALL=OK CLUSTER_ADMIN=OK PROBE_OK=4 PROBE_BAD=0 WEBPANEL=OK FAILED_UNITS=0 CLUSTER_NODES=3 QUORATE=YES GENERATED_SYNC_HEAD=597119391e7b265b85fd9a76644e4e33af47e469 DESIRED_STATE_DIRTY=0 MATRIX_SHA256=7613bcb25d32e61fac114da13af81e38354ec22a012a6deeaa168ee4cc454f87 MATRIX_STATE_SHA256=14a31112aac5d577ae01df567921b093b2fedcf7189c4cc8f009216ee0749380 CLUSTER_PASSPORT_SHA256=1b8b924ae0e5ce6872fe5671318094ce494806c2db7efe28299b5ff49adc0d0d FINAL_READINESS_SHA256=ac5159fa8c497afe021b5cb232f08cfec0e1a4d516d8aae966ba03f7034f1412 OVERALL_SHA256=6ae529bfdf5811f686d3b53978f453ce4f8757c74fc1be105de219b13a61ba46 PRESEAL_SHA256=7ea1f710cd311bd51dba05a313b4c540f57d97407f008881e502fc213b4b3c99 EVIDENCE_MANIFEST_SHA256=b4c68312026e1328b208a74090891f4809ab31ec69069e94a3d6902429bf3566 BACKUP_RUN=NO RESTORE_RUN=NO CLOUD_WRITE_RUN=NO RETENTION_RUN=NO NEXT_ACTION=AUDIT_AND_MIGRATE_SPECIAL_VM_EMERGENCY_AND_NON_APP_BACKUP_FAMILIES EVIDENCE_ROOT=/var/lib/homelab-change-sets/backup-matrix-cluster-final-seal/20260713T175131Z -HEALTH_FILE=actual-backup.txt|mtime=2026-07-09 15:53:45.248005185 +0300|sha256=72d1b31e22e22ac91af66e3d8ae46b20b3b9297809ace524eece22031458fb05 -STATUS=OK TS=2026-07-09T12:53:43Z TYPE=actual-budget-backup JOB=homelab-pve01-app-actual-budget-20260709T051713Z SOURCE=latest-appbackup SIZE=2011 LOCAL_BACKUP_DELETED=true JSON=/var/lib/homelab-backup/apps/latest-actual-budget.json NOTE=service-readiness-backup-proof -HEALTH_FILE=authentik-backup.txt|mtime=2026-08-30 21:08:08.459821635 +0300|sha256=8888712fdc8c5fd18ef0b691eb5d379fbd71af8f5dbda4e917b5a1493d12c046 -STATUS=FAIL -TS=2026-08-30T18:08:08Z -TYPE=backup-matrix-generated -JOB_ID=homelab_appbackup_postgres_batch -JOB_NAME=homelab-appbackup-postgres-batch -MATRIX_MODE=authoritative -TIMER=homelab-appbackup-postgres-batch.timer -SERVICE=homelab-appbackup-postgres-batch.service -TIMER_ENABLED=not-found -TIMER_ACTIVE=inactive -SOURCE_KIND=systemd -EVIDENCE_LEVEL=execution_only -SERVICE_RESULT=success -SERVICE_EXEC_MAIN_STATUS=0 -SOURCE_HEALTH=systemd:homelab-appbackup-postgres-batch.service -SOURCE_STATUS=OK -SOURCE_MTIME_UTC=2026-07-24T02:18:04Z -AGE_SECONDS=3253804 -MAX_AGE_SECONDS=172800 -FRESHNESS=STALE -LEGACY_SOURCE_PATH=/var/lib/homelab-health/authentik-backup.txt -CANONICAL_JOB_ID=homelab_appbackup_postgres_batch -CANONICAL_STATE=/var/lib/homelab-backup-matrix/state.json -HEALTH_FILE=cluster-admin-stage3-postapply-backup.txt|mtime=2026-07-13 06:44:42.760995774 +0300|sha256=c00feef82c76522380776c84bf5bdb742ca2105c0dc2eeb1b596db2880facef9 -STATUS=OK TS=2026-07-13T03:43:59Z TYPE=cluster-admin-stage3-postapply-backup BAD=0 COMMAND_PHYSICAL_LINES=1 PRODUCTION_CHANGED=POSTAPPLY_BACKUP_AND_CONTROLLED_COLLECTOR_RUN PRODUCTION_DATABASE_CHANGED=COLLECTOR_NORMAL_OPERATION_ONLY PRODUCTION_SCHEMA_CHANGED=NO LIVE_APPLICATION_CODE_CHANGED=NO DESIRED_STATE_CHANGED=NO SERVICE_RESTARTED=NO CLOUD_WRITE_RUN=YES_CONTROLLED_POSTAPPLY_BACKUP LIVE_APP_SHA256=a330a2d33ee5a8a57778e3541179e3faebcc0531600a504df744435e80a93b34 DESIRED_STATE_HEAD=b974057b536ece861299d0b230c06cc0e6702c39 APPBACKUP=VERIFIED APPBACKUP_JOB=homelab-pve01-app-cluster-admin-incident-engine-20260713T034400Z APPBACKUP_SIZE_BYTES=1791603 APPBACKUP_RESTORE_CHECK=OK APPBACKUP_ROUTING=VERIFIED COLLECTOR_CONTROL_RUN=OK COLLECTOR_RUN_ID=3113 CANONICAL_CONSISTENCY=0|0|0|0|0 INCIDENT_ENGINE=ACTIVE COLLECTOR_TIMER=ACTIVE VM180_FAILED_UNITS=0 READY_FOR_STAGE3_FINAL_SEAL=YES STAGE3_CLOSED=NO POSTAPPLY_ROOT=/var/lib/homelab-stage3-postapply/20260713T034359Z -HEALTH_FILE=netbird-vps-trust.txt|mtime=2026-07-18 01:54:49.485000918 +0300|sha256=643c858c4b1cfa77a7914349ccb05434a94e27a358320b687658684665d5c1ac -STATUS=OK -TS=2026-07-17T04:52:15Z -TYPE=backup-matrix-generated -JOB_ID=netbird_vps_backup -JOB_NAME=netbird-vps-backup -MATRIX_MODE=authoritative -TIMER=netbird-vps-backup.timer -SERVICE=netbird-vps-backup.service -TIMER_ENABLED=enabled -TIMER_ACTIVE=active -SOURCE_KIND=health_file -EVIDENCE_LEVEL=artifact_or_health -SERVICE_RESULT=success -SERVICE_EXEC_MAIN_STATUS=0 -SOURCE_HEALTH=/var/lib/homelab-backup-matrix/evidence/netbird_vps_backup/0-netbird-vps-trust.txt -SOURCE_STATUS=OK -SOURCE_MTIME_UTC=2026-07-07T15:24:59Z -AGE_SECONDS=826036 -MAX_AGE_SECONDS=864000 -FRESHNESS=FRESH -LEGACY_SOURCE_PATH=/var/lib/homelab-health/netbird-vps-trust.txt -CANONICAL_JOB_ID=netbird_vps_backup -CANONICAL_STATE=/var/lib/homelab-backup-matrix/state.json -HEALTH_FILE=extended-appbackup.txt|mtime=2026-08-30 21:08:08.661414778 +0300|sha256=00167921437640beadbd7c5cc7323845b3893f85c161781d0899da9fba63f991 -STATUS=FAIL -TS=2026-08-30T18:08:08Z -TYPE=backup-matrix-generated -JOB_ID=homelab_extended_appbackup -JOB_NAME=homelab-extended-appbackup -MATRIX_MODE=authoritative -TIMER=homelab-extended-appbackup.timer -SERVICE=homelab-extended-appbackup.service -TIMER_ENABLED=not-found -TIMER_ACTIVE=inactive -SOURCE_KIND=health_file -EVIDENCE_LEVEL=artifact_or_health -SERVICE_RESULT=success -SERVICE_EXEC_MAIN_STATUS=0 -SOURCE_HEALTH=/var/lib/homelab-backup-matrix/evidence/homelab_extended_appbackup/0-extended-appbackup.txt -SOURCE_STATUS=OK -SOURCE_MTIME_UTC=2026-08-06T21:34:02Z -AGE_SECONDS=2061246 -MAX_AGE_SECONDS=172800 -FRESHNESS=STALE -LEGACY_SOURCE_PATH=/var/lib/homelab-health/extended-appbackup.txt -CANONICAL_JOB_ID=homelab_extended_appbackup -CANONICAL_STATE=/var/lib/homelab-backup-matrix/state.json -HEALTH_FILE=healthchecks-backup.txt|mtime=2026-07-09 11:51:31.108386138 +0300|sha256=f2012e88374f88b98e419adb29ee39db9dd58540ed8f20d7894813eabad36301 -STATUS=OK TS=2026-07-09T08:51:29Z TYPE=healthchecks-backup JOB=homelab-pve01-app-healthchecks-20260709T051713Z SOURCE=latest-appbackup SIZE=687 LOCAL_BACKUP_DELETED=true JSON=/var/lib/homelab-backup/apps/latest-healthchecks.json NOTE=service-readiness-backup-proof -HEALTH_FILE=backup-scheduler-rpo-hardening.json|mtime=2026-08-10 13:51:49.375986623 +0300|sha256=e73927d669e1d066d03cd711449f9533ce018da041a86432c8550624a5a6c55f -{ - "status": "OK", - "root_cause_count": 0, - "production_changed": true, - "cloud_changed": false, - "backup_started": false, - "applied_at_utc": "2026-08-10T10:51:49Z", - "target_bytes": true, - "persistent_stamps_rebased": true, - "timers_started": true, - "no_immediate_service_start": true, - "next": { - "vm": "Mon 2026-08-10 [IPV6] MSK", - "app": "Mon 2026-08-10 [IPV6] MSK" - } -} -HEALTH_FILE=minio-backup.txt|mtime=2026-07-09 15:13:16.876111706 +0300|sha256=e4965b27706d848070d5ca6c2e974a14c520465f3b1575fda1ab671d6cf0e124 -STATUS=OK TS=2026-07-09T12:13:14Z TYPE=minio-backup JOB=homelab-pve01-app-minio-20260709T051713Z SOURCE=latest-appbackup SIZE=4585 LOCAL_BACKUP_DELETED=true JSON=/var/lib/homelab-backup/apps/latest-minio.json NOTE=service-readiness-backup-proof -HEALTH_FILE=backup-framework.txt|mtime=2026-08-30 21:08:08.850863900 +0300|sha256=192caeb6ecfd0d386aa32fcf838229854ea73f3abb7d46343cd8f1b4acf7b2ab -STATUS=FAIL -TS=2026-08-30T18:08:08Z -TYPE=backup-matrix-generated -JOB_ID=homelab_verified_backup_health -JOB_NAME=homelab-verified-backup-health -MATRIX_MODE=authoritative -TIMER=homelab-verified-backup-health.timer -SERVICE=homelab-verified-backup-health.service -TIMER_ENABLED=not-found -TIMER_ACTIVE=inactive -SOURCE_KIND=health_file -EVIDENCE_LEVEL=artifact_or_health -SERVICE_RESULT=success -SERVICE_EXEC_MAIN_STATUS=0 -SOURCE_HEALTH=/var/lib/homelab-backup-matrix/evidence/homelab_verified_backup_health/1-latest-emergency-pack.json -SOURCE_STATUS=OK -SOURCE_MTIME_UTC=2026-08-06T20:59:56Z -AGE_SECONDS=2063292 -MAX_AGE_SECONDS=172800 -FRESHNESS=STALE -LEGACY_SOURCE_PATH=/var/lib/homelab-health/backup-framework.txt -CANONICAL_JOB_ID=homelab_verified_backup_health -CANONICAL_STATE=/var/lib/homelab-backup-matrix/state.json -HEALTH_FILE=audiobookshelf-backup.txt|mtime=2026-07-09 21:22:25.670486930 +0300|sha256=fb81212dd08251ede49f921b0fb61b86a8f4446af80bdd42172acbbaf60b8e32 -STATUS=OK TS=2026-07-09T18:22:24Z TYPE=audiobookshelf-backup BAD=0 APP=audiobookshelf JOB=homelab-pve01-app-audiobookshelf-20260709T051713Z SIZE=23094 LOCAL_BACKUP_DELETED=true JSON=/var/lib/homelab-backup/apps/latest-audiobookshelf.json -HEALTH_FILE=filebrowser-backup.txt|mtime=2026-08-30 21:08:08.363475316 +0300|sha256=e8d06faa33f0c464b6074e5c121ec8aea9406d532097400a339764bcc71b009c -STATUS=FAIL -TS=2026-08-30T18:08:08Z -TYPE=backup-matrix-generated -JOB_ID=filebrowser_backup -JOB_NAME=filebrowser-backup -MATRIX_MODE=authoritative -TIMER=filebrowser-backup.timer -SERVICE=filebrowser-backup.service -TIMER_ENABLED=not-found -TIMER_ACTIVE=inactive -SOURCE_KIND=health_file -EVIDENCE_LEVEL=artifact_or_health -SERVICE_RESULT=success -SERVICE_EXEC_MAIN_STATUS=0 -SOURCE_HEALTH=/var/lib/homelab-backup/apps/latest-filebrowser.json -SOURCE_STATUS=OK -SOURCE_MTIME_UTC=2026-08-06T21:28:05Z -AGE_SECONDS=2061603 -MAX_AGE_SECONDS=259200 -FRESHNESS=STALE -LEGACY_SOURCE_PATH=/var/lib/homelab-health/filebrowser-backup.txt -CANONICAL_JOB_ID=filebrowser_backup -CANONICAL_STATE=/var/lib/homelab-backup-matrix/state.json -HEALTH_FILE=netbox-backup.txt|mtime=2026-07-09 13:23:55.693530989 +0300|sha256=0cf3e80b9d64332d56d5d6bb6fd0b09e3de366a6817b7cf4ce2fcca80f5ccd0c -STATUS=OK TS=2026-07-09T10:23:50Z TYPE=netbox-backup JOB=homelab-pve01-app-netbox-20260709T072101Z SOURCE=latest-appbackup SIZE=25860637 LOCAL_BACKUP_DELETED=true JSON=/var/lib/homelab-backup/apps/latest-netbox.json NOTE=service-readiness-backup-proof -HEALTH_FILE=linkding-backup.txt|mtime=2026-07-09 20:00:19.838194359 +0300|sha256=cb3766255eec88a3a0a70447fb507ced5065345636fce5c4d760012f7e7944ab -STATUS=OK TS=2026-07-09T17:00:18Z TYPE=linkding-backup BAD=0 APP=linkding JOB=homelab-pve01-app-linkding-20260709T051713Z SIZE=40380 LOCAL_BACKUP_DELETED=true JSON=/var/lib/homelab-backup/apps/latest-linkding.json -HEALTH_FILE=n8n-backup.txt|mtime=2026-07-09 22:39:24.237551932 +0300|sha256=723af1f45fc9e9e68e95cccc9747fb25b6bfef068c71060de1aaf260c0534774 -STATUS=OK TS=2026-07-09T19:39:23Z TYPE=n8n-backup BAD=0 APP=n8n JOB=homelab-pve01-app-n8n-20260709T051713Z SIZE=92667 LOCAL_BACKUP_DELETED=true JSON=/var/lib/homelab-backup/apps/latest-n8n.json -HEALTH_FILE=crypto-portfolio-backup.txt|mtime=2026-07-13 11:12:13.302397603 +0300|sha256=cf13d8b15a64fbb2fa412e32ce4eafdc732b045fe82e789928e768ace92361eb -STATUS=OK TS=2026-07-13T08:12:13Z TYPE=crypto-portfolio-backup APP=crypto-portfolio JOB=homelab-pve01-app-crypto-portfolio-20260713T075801Z PATH_REL=hb/a/cp-260713-075801 ARCHIVE_SIZE_BYTES=137496 FILE_COUNT=19 ARCHIVE_SHA256=d6d3f6a397c5e73b1df0ceaeeef0f313db557b163bc6b36f32e8a084e54e975a DATABASE_INTEGRITY=ok TWO_CLOUDS=OK LOCAL_STAGING_DELETED=YES -HEALTH_FILE=backup-audit-v2.json|mtime=2026-09-23 23:47:36.447938462 +0300|sha256=b193b9fc04ed47f247d4c981468387ceff69cf1415e4295ec5b4de9e08d1b6fe -{"cloud":{"A":{"free":592249131673,"free_fraction":0.538647,"readable":true,"top_dirs":27,"total":1099511627776,"used":507262496103},"B":{"free":592249067814,"free_fraction":0.538647,"readable":true,"top_dirs":27,"total":1099511627776,"used":507262559962}},"cloud_b_restore_drill":true,"dr_proven":true,"generated_at_epoch":1790196452,"issues":[{"code":"RPO_BREACH","logical_id":"vm150","root_cause":"NO_FRESH_V2_OR_SEALED_QUORUM","severity":"HIGH"},{"code":"SOURCE_OFFHOST_MISSING","logical_id":"-","root_cause":"CR_BRANCH_SHA_NOT_EXACT","severity":"MEDIUM"}],"schema":"homelab-backup-audit-v3","source":{"local_head":"6401ea1e99988b9d2550de544c9e45bd7e829bdd","pushurl_disabled":true,"remote_head":null},"status":"RED","targets":[{"decision":"SKIP_FRESH","generation":"app-filebrowser-pve01-20260923T143005Z","kind":"APP","legacy_age_seconds":22640,"logical_id":"app-filebrowser","state":"FRESH"},{"decision":"SKIP_FRESH","generation":"app-hbarhub-pve01-20260923T143158Z","kind":"APP","legacy_age_seconds":22468,"logical_id":"app-hbarhub","state":"FRESH"},{"decision":"SKIP_FRESH","generation":"app-immich-pve01-20260923T143041Z","kind":"APP","legacy_age_seconds":22568,"logical_id":"app-immich","state":"FRESH"},{"decision":"SKIP_FRESH","generation":"app-skladchik-reports-monitor-pve01-20260923T143025Z","kind":"APP","legacy_age_seconds":22618,"logical_id":"app-skladchik-reports-monitor","state":"FRESH"},{"age_seconds":44470,"generation":"control-bootstrap-v2-pve01-20260916T120057Z","kind":"CONTROL","logical_id":"control-bootstrap-v2","state":"FRESH"},{"generation":"v2-ct110-20260923T125546Z-34408ffe4182","kind":"VM","legacy_age_seconds":1696083,"logical_id":"vm110","replica_count":2,"source":"V2","state":"FRESH","state_id":"ct110","v2_age_seconds":28097},{"generation":"v2-ct111-20260923T133222Z-0d3f5fae9c35","kind":"VM","legacy_age_seconds":null,"logical_id":"vm111","replica_count":2,"source":"V2","state":"FRESH","state_id":"ct111","v2_age_seconds":25955},{"generation":"v2-ct112-20260923T200600Z-c1b47cc67dad","kind":"VM","legacy_age_seconds":1734578,"logical_id":"vm112","replica_count":2,"source":"V2","state":"FRESH","state_id":"ct112","v2_age_seconds":2422},{"generation":"v2-ct113-20260923T203800Z-bd913a8e8930","kind":"VM","legacy_age_seconds":null,"logical_id":"vm113","replica_count":2,"source":"V2","state":"FRESH","state_id":"ct113","v2_age_seconds":497},{"generation":"v2-vm130-20260923T011558Z-c9498d37d3d7","kind":"VM","legacy_age_seconds":null,"logical_id":"vm130","replica_count":2,"source":"V2","state":"FRESH","state_id":"vm130","v2_age_seconds":69146},{"generation":"vm150-pve01-20260904T053517Z","kind":"VM","legacy_age_seconds":1696199,"logical_id":"vm150","replica_count":2,"source":"NONE","state":"STALE","state_id":"vm150","v2_age_seconds":74828},{"generation":"v2-vm160-20260923T014608Z-f76d7671af5a","kind":"VM","legacy_age_seconds":null,"logical_id":"vm160","replica_count":2,"source":"V2","state":"FRESH","state_id":"vm160","v2_age_seconds":68046},{"generation":"v2-vm170-20260923T035958Z-923adacbc94b","kind":"VM","legacy_age_seconds":640820,"logical_id":"vm170","replica_count":2,"source":"V2","state":"FRESH","state_id":"vm170","v2_age_seconds":59695},{"generation":"v2-vm171-20260923T021946Z-4d6e4f578075","kind":"VM","legacy_age_seconds":670352,"logical_id":"vm171","replica_count":2,"source":"V2","state":"FRESH","state_id":"vm171","v2_age_seconds":66249},{"generation":"v2-vm190-20260923T032759Z-314d9ba2ec35","kind":"VM","legacy_age_seconds":null,"logical_id":"vm190","replica_count":2,"source":"V2","state":"FRESH","state_id":"vm190","v2_age_seconds":62166},{"generation":"v2-vm9130-20260923T071932Z-c0492db6535b","kind":"VM","legacy_age_seconds":null,"logical_id":"vm9130","replica_count":2,"source":"V2","state":"FRESH","state_id":"vm9130","v2_age_seconds":47261}]} -HEALTH_FILE=netbird-old-usa-poweroff-observation.txt|mtime=2026-08-22 05:33:01.774623609 +0300|sha256=0307f121dba0190d02875321ed3ca2c433f8a2902678f46814d39cdc476a4a00 -STATUS=OBSERVING -START_EPOCH=1787365981 -START_UTC=2026-08-22T02:33:01Z -MIN_SECONDS=86400 -MAX_SECONDS=172800 -NEW_IP=[IP] -OLD_USA_PROVIDER_DELETION=USER_ATTESTED -OLD_PUBLIC_IP_REACHABILITY=NOT_USED_AS_HOST_IDENTITY -NEW_OPERATOR_KEY_ONLY=PROVEN -NEW_NETBIRD_STACK=3_OF_3 -NEW_PUBLIC_PLANE=GREEN -FINAL_RETIREMENT=PENDING_OBSERVATION -HEALTH_FILE=crypto-portfolio-remove-grt-prechange-backup.txt|mtime=2026-07-13 08:10:44.727781612 +0300|sha256=59de8a4e38310ef8236d167be536aa2f0f1d0fa3dd0ad3ac7fd49e67ed6206f3 -STATUS=OK TS=2026-07-13T05:10:13Z TYPE=crypto-portfolio-remove-grt-prechange-backup BAD=0 COMMAND_PHYSICAL_LINES=1 PRODUCTION_CHANGED=ROLLBACK_SNAPSHOT_AND_BACKUP_ONLY PRODUCTION_DATABASE_CHANGED=NO DESIRED_STATE_CHANGED=NO_BY_THIS_COMMAND SERVICE_RESTARTED=NO PRODUCTION_CONTAINER_RECREATED=NO CLOUD_WRITE_RUN=YES_CONTROLLED_PRECHANGE_BACKUP RETENTION_RUN=NO PRODUCTION_APPLY=NO TARGET_HOST=core-apps.gram1.ru DESIRED_STATE_HEAD=80763a6c62b94cde4f3293f3b529980f76919d25 CANDIDATE_ROOT=/var/lib/homelab-change-sets/crypto-portfolio-remove-grt-candidate-v3/20260713T042046Z INPUT_APP_SHA256=684ad884119554503db0ed36252aeb017c0c04b54729f30c51e66a7ab293f8bd INPUT_WORKER_SHA256=40a1c81111d7097120125b140683a8604abcb4051db5073ab8af708a11854038 INPUT_INDEX_SHA256=2b1ab459cabf3050af1db4a7b293fff7c2a9ac18bf4a14faa1efc70a3a3f63e3 EXACT_LOCAL_ROLLBACK=OK ROLLBACK_PATH=/var/lib/homelab-incidents/crypto-remove-grt-prechange-20260713T051013Z ROLLBACK_MANIFEST_SHA256=6cc505f7e13a02b0ddfc9fd3a9700903d7aaf70d8c8ae3f630a4fea69aeb4b1f ROLLBACK_DATABASE_INTEGRITY=ok ROLLBACK_GRT_ROWS=1|1|729 TWO_CLOUD_BACKUP=OK BACKUP_JOB=homelab-pve01-app-crypto-portfolio-20260713T051015Z BACKUP_ARCHIVE_SIZE_BYTES=172776 BACKUP_FILE_COUNT=19 BACKUP_DATABASE_INTEGRITY=ok BACKUP_LOCAL_STAGING_DELETED=YES RESTORE_CHECK=OK_BOTH_CLOUDS PRODUCTION_SOURCE_HASHES_UNCHANGED=YES PRODUCTION_CONTAINER_IDS_AND_RESTARTS_UNCHANGED=YES PRODUCTION_DATABASE_INTEGRITY=ok PRODUCTION_GRT_STATE_PRESERVED=YES PUBLIC_PRODUCTION_STATE_PRESERVED=YES READY_FOR_GRT_PRODUCTION_APPLY=YES PRECHANGE_ROOT=/var/lib/homelab-change-sets/crypto-portfolio-remove-grt-prechange-backup/20260713T051013Z -HEALTH_FILE=beszel-backup.txt|mtime=2026-07-09 11:38:36.118613080 +0300|sha256=951a25a876714a6b36934a96ae7f85bab06b480633c052b0850ba02d5e0e106c -STATUS=OK TS=2026-07-09T08:38:10Z TYPE=beszel-backup JOB=homelab-pve01-app-beszel-20260709T083810Z SOURCE=beszel_stack_data SIZE=20030 SHA256=a3d7069dce01b7e7b336268cfe383dc1b0842a974bd5dc9f455442ed86c912e4 LOCAL_BACKUP_DELETED=true NOTE=service-readiness-backup-proof -HEALTH_FILE=gotify-backup.txt|mtime=2026-07-09 23:39:37.911441599 +0300|sha256=e86e933ad71df820106148796cdccbde701c986034c7e49684f3d9e84bc0d17f -STATUS=OK TS=2026-07-09T20:39:29Z TYPE=gotify-backup BAD=0 APP=gotify JOB=homelab-pve01-app-gotify-20260709T072202Z SIZE=44431 LOCAL_BACKUP_DELETED=true JSON=/var/lib/homelab-backup/apps/latest-gotify.json -HEALTH_FILE=minio-console-backup.txt|mtime=2026-07-09 15:13:16.876111706 +0300|sha256=e4965b27706d848070d5ca6c2e974a14c520465f3b1575fda1ab671d6cf0e124 -STATUS=OK TS=2026-07-09T12:13:14Z TYPE=minio-backup JOB=homelab-pve01-app-minio-20260709T051713Z SOURCE=latest-appbackup SIZE=4585 LOCAL_BACKUP_DELETED=true JSON=/var/lib/homelab-backup/apps/latest-minio.json NOTE=service-readiness-backup-proof -HEALTH_FILE=netbird-vps-restore.txt|mtime=2026-06-17 10:54:28.050678269 +0300|sha256=800a5462a6173c8749059462e39a3263cae2b4db012c1dc3ad336aa3429ab2cb -STATUS=OK -SERVICE=netbird-vps-restore-validation -TIMESTAMP_UTC=20260617T075428Z -LATEST=20260617T074922Z -ARCHIVE=/mnt/staging/netbird-vps-backups/snapshots/20260617T074922Z/netbird-vps-backup.tgz -SIZE=32M -SHA256=8e0c6538268b4e5dc463e008e59acb9d77873873915bec74ac2a85a9d1523a84 -TOP_ARCHIVE_ENTRIES=9 -ROOT_NETBIRD_ENTRIES=7 -CHECKS=compose,config,dashboard_env -HEALTH_FILE=bookstack-backup.txt|mtime=2026-07-09 20:58:33.528404671 +0300|sha256=80a1b54e2027c39a6b2c298e558d1352b976e0cd1795cf8f26ae9249b8c2e1dd -STATUS=OK TS=2026-07-09T17:58:30Z TYPE=bookstack-backup BAD=0 APP=bookstack JOB=homelab-pve01-app-bookstack-20260709T024714Z SIZE=18108 LOCAL_BACKUP_DELETED=true JSON=/var/lib/homelab-backup/apps/latest-bookstack.json -HEALTH_FILE=backup-last-run.log|mtime=2026-07-24 07:04:41.713444052 +0300|sha256=8618b97d2ffa65b7cc2218710c2add4283285038eaca633a7bffbda898d1e5c7 -===== HOMELAB BACKUP HEALTHCHECK ===== -2026-07-24T07:04:28+03:00 - -LATEST=/mnt/staging/vzdump/auto-full-backup-20260719T004537Z -AGE_HOURS=123 -MAX_AGE_HOURS=192 - -===== REQUIRED FILES ===== -OK: pve01/vzdump-lxc-110-*.tar.zst -OK: pve01/vzdump-lxc-112-*.tar.zst -OK: pve01/vzdump-qemu-150-*.vma.zst -OK: pve02/vzdump-lxc-111-*.tar.zst -OK: pve02/vzdump-lxc-113-*.tar.zst -OK: pve03/vzdump-qemu-130-*.vma.zst -OK: MASTER_SHA256SUMS.txt -OK: README_AUTO_FULL_BACKUP.txt - -===== SIZE CHECK ===== -SIZE_BYTES=19102676192 -SIZE_GB_APPROX=17 - -===== SHA256 CHECK ===== -pve01/SHA256SUMS.txt: OK -pve01/vzdump-lxc-110-2026_07_19-03_45_38.log: OK -pve01/vzdump-lxc-110-2026_07_19-03_45_38.tar.zst: OK -pve01/vzdump-lxc-112-2026_07_19-03_45_47.log: OK -pve01/vzdump-lxc-112-2026_07_19-03_45_47.tar.zst: OK -pve01/vzdump-qemu-150-2026_07_19-03_45_58.log: OK -pve01/vzdump-qemu-150-2026_07_19-03_45_58.vma.zst: OK -pve02/SHA256SUMS.txt: OK -pve02/vzdump-lxc-111-2026_07_19-03_47_26.log: OK -pve02/vzdump-lxc-111-2026_07_19-03_47_26.tar.zst: OK -pve02/vzdump-lxc-113-2026_07_19-03_47_43.log: OK -pve02/vzdump-lxc-113-2026_07_19-03_47_43.tar.zst: OK -pve03/SHA256SUMS.txt: OK -pve03/vzdump-qemu-130-2026_07_19-03_48_01.log: OK -pve03/vzdump-qemu-130-2026_07_19-03_48_01.vma.zst: OK -README_AUTO_FULL_BACKUP.txt: OK - -===== STORAGE SPACE ===== -Filesystem Type Size Used Avail Use% Mounted on -/dev/mapper/pve-homelab--staging ext4 295G 148G 132G 53% /mnt/staging -/dev/mapper/pve-root ext4 98G 41G 53G 44% / - -STATUS=OK -REASON=latest backup exists, fresh, complete, and sha256 verified -HEALTH_FILE=cluster-admin-stage2-preapply-backup.txt|mtime=2026-07-13 04:24:04.995755658 +0300|sha256=82a0a705c46b2cf757dc37ce6a06a981c4cbe3ea3d580459ac7f8c6ae17a153d -STATUS=OK TS=2026-07-13T01:23:18Z TYPE=cluster-admin-stage2-preapply-backup BAD=0 PREAPPLY_GENERATION=V2_DYNAMIC PRODUCTION_CHANGED=BACKUP_ARTIFACTS_ONLY PRODUCTION_DATABASE_CHANGED=NO PRODUCTION_SCHEMA_CHANGED=NO INCIDENT_ENGINE_RESTARTED=NO COLLECTOR_ONESHOT_RUN=NO CLOUD_WRITE_RUN=YES_CONTROLLED_BACKUP_ONLY ROLLBACK_POINT=VERIFIED DIRECT_DB_SNAPSHOT=VERIFIED DIRECT_DB_RESTORE_TEST=OK DIRECT_DB_DUMP_SIZE_BYTES=1818083 DIRECT_DB_DUMP_SHA256=c0fe2161bb6e61b423bc72d809bdd579ac29dc4f7bac79ba4c493bfa9f8b1d5e PRECHANGE_COLLECTOR_SHA256=a83abde2290d8392b1c631122671bcb680f9271d0b576fafacb8876d3c8034f1 CANDIDATE_COLLECTOR_SHA256=5dda66a1878046223af9c28a393129b44991a8083b430a76302767b460e7cbbe FOUNDATION_COUNTS=84|21|1|227|26|0 DYNAMIC_INVARIANTS=0|0|21|21|0|0|0|0|0|0 RETIRED_PLACEHOLDERS_ALLOWED=YES FIXED_TOTAL_ENTITIES_REQUIRED=NO FIXED_TOTAL_ALIASES_REQUIRED=NO APPBACKUP_JOB=homelab-pve01-app-cluster-admin-incident-engine-20260713T012320Z APPBACKUP_STATUS=success APPBACKUP_RESTORE_CHECK=OK APPBACKUP_ROUTING_DOCKGE_UNCHANGED=YES MIGRATION_PATCH_REQUIRED=NO COLLECTOR_PATCH_REQUIRED=NO APPLY_VERIFIER_PATCH_REQUIRED=YES PREFLIGHT_TRANSACTION_ROLLBACK_REQUIRED=YES ROLLBACK_AFTER_COMMITTED_SYNC=PRESERVE_APPLIED_STATE_AND_DIAGNOSE PRODUCTION_UNCHANGED=YES OVERALL=OK DUTY_ADMIN=OK READY_FOR_DYNAMIC_PRODUCTION_APPLY=YES DESIRED_STATE_HEAD=4d0d03395d2baf4024ad7c9d5870f21bdaee9f99 PREAPPLY_ROOT=/var/lib/homelab-stage2-preapply-v2/20260713T012318Z REMOTE_SNAPSHOT_DIR=/var/lib/cluster-admin-stage2-preapply-v2/20260713T012318Z CONTRACT=/var/lib/homelab-change-sets/stage2-dynamic-redryrun/20260713T012006Z/dynamic-acceptance-contract.json EVIDENCE_MANIFEST=/var/lib/homelab-stage2-preapply-v2/20260713T012318Z/SHA256SUMS -HEALTH_FILE=blackbox-exporter-two-cloud-backup.txt|mtime=2026-07-10 22:20:26.139732318 +0300|sha256=ab822f689cc4b7c38a0eed3c6a3164993ceb54fa05033250c68d934d3df0f259 -STATUS=OK TS=2026-07-10T19:20:26Z TYPE=blackbox-exporter-two-cloud-backup BAD=0 JOB=homelab-pve01-app-blackbox-exporter-20260710T191958Z ARCHIVE_SIZE_BYTES=592 LOCAL_STAGING_DELETED=true OFFHOST=OK RESTORE=OK -HEALTH_FILE=backup.json|mtime=2026-07-24 07:04:41.723802122 +0300|sha256=550e7a326c46e3e765a6db01da2ddcaddd83808adfa638d980685286bebb45cb -{ - "status": "OK", - "reason": "latest backup exists, fresh, complete, and sha256 verified", - "latest": "/mnt/staging/vzdump/auto-full-backup-20260719T004537Z", - "age_hours": "123", - "start": "2026-07-24T07:04:28+03:00", - "end": "2026-07-24T07:04:41+03:00", - "rc": 0 -} -HEALTH_FILE=cluster-admin-backup-webpanel-final-seal.txt|mtime=2026-07-09 08:24:54.072015305 +0300|sha256=9739e2a0d2bbb859b25255fd7acae984ee05a16fb79cd9e59ff81529c4acf3fd -STATUS=OK TS=2026-07-09T05:24:54Z TYPE=cluster-admin-backup-webpanel-final-seal BAD=0 VMID=180 URL=http://[IP]:8080/ BACKUP=MAIL_CLOUD_2COPY_VERIFY_OK WEBPANEL=OK WEBPANEL_SYNC=OK OVERALL_HARDGATE=YES FINAL_READINESS_HARDGATE=YES FAILED_UNITS=pve01:0,pve02:0,pve03:0,edge:0 DIRTY_COUNT=0 REPORT=/var/lib/homelab-ideal-cluster/cluster-admin-backup-webpanel-final-seal-latest.md -HEALTH_FILE=immich-backup.txt|mtime=2026-07-09 23:09:25.074550210 +0300|sha256=8ef28f515f12888a7b1ff5b949d09b0eef90c1fc11da523b388ddd905167b52d -STATUS=OK TS=2026-07-09T20:09:12Z TYPE=immich-backup BAD=0 APP=immich JOB=homelab-pve01-app-immich-20260709T072002Z SIZE=328392286 LOCAL_BACKUP_DELETED=true JSON=/var/lib/homelab-backup/apps/latest-immich.json MEDIA_BACKUP_HEALTH=/var/lib/homelab-health/immich-media-backup.txt -HEALTH_FILE=grafana-backup.txt|mtime=2026-07-09 10:46:00.905608392 +0300|sha256=1347e0bba67dd249caa8d4e46faf4d6bb2fd573255c1f78449995f97eea30014 -STATUS=OK TS=2026-07-09T07:46:00Z TYPE=grafana-backup JOB=homelab-pve01-app-grafana-20260709T074600Z SOURCE=grafana_sqlite_backup_plus_provisioning SIZE=129929 SHA256=582537912f306666f04d9e855491f7275bdb31b39b4c9b0156444a297f8c1641 LOCAL_BACKUP_DELETED=true NOTE=service-readiness-backup-proof -HEALTH_FILE=appbackupctl-routing.txt|mtime=2026-07-13 03:29:40.002577636 +0300|sha256=d8ddbdb46b42bcefb4f03b50ef4b88bdc5e3cd1cea087dd035fd90fd78e1a252 -STATUS=OK TS=2026-07-13T00:29:36Z TYPE=appbackupctl-routing BAD=0 RESULT=APPLIED PRODUCTION_CHANGED=BACKUP_CLI_ROUTING_AND_DESIRED_STATE_ONLY PRODUCTION_DATABASE_CHANGED=NO PRODUCTION_SCHEMA_CHANGED=NO SERVICE_RESTARTED=NO CLOUD_WRITE_RUN=NO DEFAULT_ROUTE=dockge-stacks CLUSTER_ADMIN_ROUTE=homelab-cluster-admin-incident-engine-backup.service UNSUPPORTED_TARGET_POLICY=REFUSE_RC_2 ROUTE_TEST_DOCKGE=OK ROUTE_TEST_CLUSTER_ADMIN=OK ROUTE_TEST_DEFAULT=OK ROUTE_TEST_UNSUPPORTED=OK FRAMEWORK_CHECK=OK UNSUPPORTED_ROUTE_BACKUP_CREATED=NO LIVE_MATCH_DESIRED_STATE=YES PVE01_FAILED_UNITS=0 OVERALL=OK DUTY_ADMIN=OK STAGE1_CLOSED=YES STAGE2_READY=YES OLD_HEAD=bd5eb856a8d61a5fb83c2bcaf74cb8ac98f075c5 NEW_HEAD=4d0d03395d2baf4024ad7c9d5870f21bdaee9f99 CHANGE_SET=/var/lib/homelab-change-sets/appbackupctl-routing/20260713T002936Z -HEALTH_FILE=vaultwarden-backup.txt|mtime=2026-08-30 21:08:08.480614603 +0300|sha256=ba016970c3d244955b82977178290fea97126dbcc2a44e4758ecd28ad720a499 -STATUS=FAIL -TS=2026-08-30T18:08:08Z -TYPE=backup-matrix-generated -JOB_ID=homelab_appbackup_sqlite_batch -JOB_NAME=homelab-appbackup-sqlite-batch -MATRIX_MODE=authoritative -TIMER=homelab-appbackup-sqlite-batch.timer -SERVICE=homelab-appbackup-sqlite-batch.service -TIMER_ENABLED=not-found -TIMER_ACTIVE=inactive -SOURCE_KIND=systemd -EVIDENCE_LEVEL=execution_only -SERVICE_RESULT=success -SERVICE_EXEC_MAIN_STATUS=0 -SOURCE_HEALTH=systemd:homelab-appbackup-sqlite-batch.service -SOURCE_STATUS=OK -SOURCE_MTIME_UTC=2026-07-24T01:46:28Z -AGE_SECONDS=3255700 -MAX_AGE_SECONDS=172800 -FRESHNESS=STALE -LEGACY_SOURCE_PATH=/var/lib/homelab-health/vaultwarden-backup.txt -CANONICAL_JOB_ID=homelab_appbackup_sqlite_batch -CANONICAL_STATE=/var/lib/homelab-backup-matrix/state.json -HEALTH_FILE=hapusya-site-preapply-backup.txt|mtime=2026-07-13 14:03:06.015223430 +0300|sha256=00e05707281a8d6e841ec0f6d2b8c95ee0f9fbc090ab90b718e85530d41854f8 -STATUS=OK TS=2026-07-13T11:03:06Z TYPE=hapusya-site-preapply-backup BAD=0 NPMPLUS_JOB=homelab-pve01-app-npmplus-edge-20260713T110238Z TWO_CLOUDS=OK RESTORE_CHECK=OK ROLLBACK=/var/lib/homelab-incidents/hapusya-site-preapply-20260713T110305Z ROLLBACK_MANIFEST_SHA256=6b4020425c5d6c581c48a5dc95b20d54fd7906468e4ba8d27c0f0325e2df81a3 CANDIDATE_BROWSER=OK BROWSER_RUNS=4 PRODUCTION_CHANGED=NO -HEALTH_FILE=pve01-backup.txt|mtime=2026-07-09 15:29:41.985138298 +0300|sha256=2b8ca75eeec121d7e2b0c6a309d43798da5f864b57fef327f37a56b9ced3b5a8 -STATUS=OK TS=2026-07-09T12:29:38Z TYPE=proxmox-pve01-backup JOB=homelab-pve01-app-proxmox-pve01-20260709T122938Z SOURCE=safe_pve01_config_facts SIZE=4314 LOCAL_BACKUP_DELETED=true JSON=/var/lib/homelab-backup/apps/latest-proxmox-pve01.json NOTE=service-readiness-backup-proof -HEALTH_FILE=backup-observability-history.jsonl|mtime=2026-09-21 07:07:33.804291373 +0300|sha256=24fa8197552d64778c83071fb94a64accd398075ce5d1e675dcdf6c97a84c36b -{"clouds":[{"free_bytes":792267937635,"remote":"pve01-mail-01-crypt","total_bytes":1099511627776,"used_bytes":307243690141},{"free_bytes":792267873776,"remote":"pve01-mail-02-crypt","total_bytes":1099511627776,"used_bytes":307243754000}],"observed_at_epoch":1788438952,"schema":"homelab-backup-observability-snapshot-v1","targets":[{"archive_size_bytes":474143241,"id":"110","kind":"VM","logical_id":"vm110"},{"archive_size_bytes":369191552,"id":"111","kind":"VM","logical_id":"vm111"},{"archive_size_bytes":223627414,"id":"112","kind":"VM","logical_id":"vm112"},{"archive_size_bytes":221521137,"id":"113","kind":"VM","logical_id":"vm113"},{"archive_size_bytes":10622796310,"id":"130","kind":"VM","logical_id":"vm130"},{"archive_size_bytes":890940072,"id":"150","kind":"VM","logical_id":"vm150"},{"archive_size_bytes":3469023634,"id":"160","kind":"VM","logical_id":"vm160"},{"archive_size_bytes":7824247513,"id":"170","kind":"VM","logical_id":"vm170"},{"archive_size_bytes":2026787893,"id":"171","kind":"VM","logical_id":"vm171"},{"archive_size_bytes":1176306536,"id":"180","kind":"VM","logical_id":"vm180"},{"archive_size_bytes":1551688736,"id":"190","kind":"VM","logical_id":"vm190"},{"archive_size_bytes":11005893463,"id":"9130","kind":"VM","logical_id":"vm9130"},{"archive_size_bytes":383573170,"id":"200","kind":"VM","logical_id":"vm200"},{"archive_size_bytes":3193,"id":"filebrowser","kind":"APP","logical_id":"app-filebrowser"},{"archive_size_bytes":152745,"id":"skladchik-reports-monitor","kind":"APP","logical_id":"app-skladchik-reports-monitor"},{"archive_size_bytes":270193536,"id":"immich","kind":"APP","logical_id":"app-immich"},{"archive_size_bytes":75298798,"id":"hbarhub","kind":"APP","logical_id":"app-hbarhub"},{"archive_size_bytes":8959200,"id":"control-bootstrap-v2","kind":"CONTROL","logical_id":"control-bootstrap-v2"}]} -{"clouds":[{"free_bytes":792267937635,"remote":"pve01-mail-01-crypt","total_bytes":1099511627776,"used_bytes":307243690141},{"free_bytes":792267873776,"remote":"pve01-mail-02-crypt","total_bytes":1099511627776,"used_bytes":307243754000}],"observed_at_epoch":1788442632,"schema":"homelab-backup-observability-snapshot-v1","targets":[{"archive_size_bytes":474143241,"id":"110","kind":"VM","logical_id":"vm110"},{"archive_size_bytes":369191552,"id":"111","kind":"VM","logical_id":"vm111"},{"archive_size_bytes":223627414,"id":"112","kind":"VM","logical_id":"vm112"},{"archive_size_bytes":221521137,"id":"113","kind":"VM","logical_id":"vm113"},{"archive_size_bytes":10622796310,"id":"130","kind":"VM","logical_id":"vm130"},{"archive_size_bytes":890940072,"id":"150","kind":"VM","logical_id":"vm150"},{"archive_size_bytes":3469023634,"id":"160","kind":"VM","logical_id":"vm160"},{"archive_size_bytes":7824247513,"id":"170","kind":"VM","logical_id":"vm170"},{"archive_size_bytes":2026787893,"id":"171","kind":"VM","logical_id":"vm171"},{"archive_size_bytes":1176306536,"id":"180","kind":"VM","logical_id":"vm180"},{"archive_size_bytes":1551688736,"id":"190","kind":"VM","logical_id":"vm190"},{"archive_size_bytes":11005893463,"id":"9130","kind":"VM","logical_id":"vm9130"},{"archive_size_bytes":383573170,"id":"200","kind":"VM","logical_id":"vm200"},{"archive_size_bytes":3193,"id":"filebrowser","kind":"APP","logical_id":"app-filebrowser"},{"archive_size_bytes":152745,"id":"skladchik-reports-monitor","kind":"APP","logical_id":"app-skladchik-reports-monitor"},{"archive_size_bytes":270193536,"id":"immich","kind":"APP","logical_id":"app-immich"},{"archive_size_bytes":75298798,"id":"hbarhub","kind":"APP","logical_id":"app-hbarhub"},{"archive_size_bytes":8959200,"id":"control-bootstrap-v2","kind":"CONTROL","logical_id":"control-bootstrap-v2"}]} -{"clouds":[{"free_bytes":792267937635,"remote":"pve01-mail-01-crypt","total_bytes":1099511627776,"used_bytes":307243690141},{"free_bytes":792267873776,"remote":"pve01-mail-02-crypt","total_bytes":1099511627776,"used_bytes":307243754000}],"observed_at_epoch":1788447976,"schema":"homelab-backup-observability-snapshot-v1","targets":[{"archive_size_bytes":474143241,"id":"110","kind":"VM","logical_id":"vm110"},{"archive_size_bytes":369191552,"id":"111","kind":"VM","logical_id":"vm111"},{"archive_size_bytes":223627414,"id":"112","kind":"VM","logical_id":"vm112"},{"archive_size_bytes":221521137,"id":"113","kind":"VM","logical_id":"vm113"},{"archive_size_bytes":10622796310,"id":"130","kind":"VM","logical_id":"vm130"},{"archive_size_bytes":890940072,"id":"150","kind":"VM","logical_id":"vm150"},{"archive_size_bytes":3469023634,"id":"160","kind":"VM","logical_id":"vm160"},{"archive_size_bytes":7824247513,"id":"170","kind":"VM","logical_id":"vm170"},{"archive_size_bytes":2026787893,"id":"171","kind":"VM","logical_id":"vm171"},{"archive_size_bytes":1176306536,"id":"180","kind":"VM","logical_id":"vm180"},{"archive_size_bytes":1551688736,"id":"190","kind":"VM","logical_id":"vm190"},{"archive_size_bytes":11005893463,"id":"9130","kind":"VM","logical_id":"vm9130"},{"archive_size_bytes":383573170,"id":"200","kind":"VM","logical_id":"vm200"},{"archive_size_bytes":3193,"id":"filebrowser","kind":"APP","logical_id":"app-filebrowser"},{"archive_size_bytes":152745,"id":"skladchik-reports-monitor","kind":"APP","logical_id":"app-skladchik-reports-monitor"},{"archive_size_bytes":270193536,"id":"immich","kind":"APP","logical_id":"app-immich"},{"archive_size_bytes":75298798,"id":"hbarhub","kind":"APP","logical_id":"app-hbarhub"},{"archive_size_bytes":8959200,"id":"control-bootstrap-v2","kind":"CONTROL","logical_id":"control-bootstrap-v2"}]} -{"clouds":[{"free_bytes":792267937635,"remote":"pve01-mail-01-crypt","total_bytes":1099511627776,"used_bytes":307243690141},{"free_bytes":792267873776,"remote":"pve01-mail-02-crypt","total_bytes":1099511627776,"used_bytes":307243754000}],"observed_at_epoch":1788451586,"schema":"homelab-backup-observability-snapshot-v1","targets":[{"archive_size_bytes":474143241,"id":"110","kind":"VM","logical_id":"vm110"},{"archive_size_bytes":369191552,"id":"111","kind":"VM","logical_id":"vm111"},{"archive_size_bytes":223627414,"id":"112","kind":"VM","logical_id":"vm112"},{"archive_size_bytes":221521137,"id":"113","kind":"VM","logical_id":"vm113"},{"archive_size_bytes":10622796310,"id":"130","kind":"VM","logical_id":"vm130"},{"archive_size_bytes":890940072,"id":"150","kind":"VM","logical_id":"vm150"},{"archive_size_bytes":3469023634,"id":"160","kind":"VM","logical_id":"vm160"},{"archive_size_bytes":7824247513,"id":"170","kind":"VM","logical_id":"vm170"},{"archive_size_bytes":2026787893,"id":"171","kind":"VM","logical_id":"vm171"},{"archive_size_bytes":1176306536,"id":"180","kind":"VM","logical_id":"vm180"},{"archive_size_bytes":1551688736,"id":"190","kind":"VM","logical_id":"vm190"},{"archive_size_bytes":11005893463,"id":"9130","kind":"VM","logical_id":"vm9130"},{"archive_size_bytes":383573170,"id":"200","kind":"VM","logical_id":"vm200"},{"archive_size_bytes":3193,"id":"filebrowser","kind":"APP","logical_id":"app-filebrowser"},{"archive_size_bytes":152745,"id":"skladchik-reports-monitor","kind":"APP","logical_id":"app-skladchik-reports-monitor"},{"archive_size_bytes":270193536,"id":"immich","kind":"APP","logical_id":"app-immich"},{"archive_size_bytes":75298798,"id":"hbarhub","kind":"APP","logical_id":"app-hbarhub"},{"archive_size_bytes":8959200,"id":"control-bootstrap-v2","kind":"CONTROL","logical_id":"control-bootstrap-v2"}]} -{"clouds":[{"free_bytes":771612988784,"remote":"pve01-mail-01-crypt","total_bytes":1099511627776,"used_bytes":327898638992},{"free_bytes":771612924925,"remote":"pve01-mail-02-crypt","total_bytes":1099511627776,"used_bytes":327898702851}],"observed_at_epoch":1788456981,"schema":"homelab-backup-observability-snapshot-v1","targets":[{"archive_size_bytes":476345840,"id":"110","kind":"VM","logical_id":"vm110"},{"archive_size_bytes":369191552,"id":"111","kind":"VM","logical_id":"vm111"},{"archive_size_bytes":223627414,"id":"112","kind":"VM","logical_id":"vm112"},{"archive_size_bytes":221521137,"id":"113","kind":"VM","logical_id":"vm113"},{"archive_size_bytes":892033158,"id":"150","kind":"VM","logical_id":"vm150"},{"archive_size_bytes":3469023634,"id":"160","kind":"VM","logical_id":"vm160"},{"archive_size_bytes":7937058725,"id":"170","kind":"VM","logical_id":"vm170"},{"archive_size_bytes":2026787893,"id":"171","kind":"VM","logical_id":"vm171"},{"archive_size_bytes":1176306536,"id":"180","kind":"VM","logical_id":"vm180"},{"archive_size_bytes":1551688736,"id":"190","kind":"VM","logical_id":"vm190"},{"archive_size_bytes":11005893463,"id":"9130","kind":"VM","logical_id":"vm9130"},{"archive_size_bytes":386825665,"id":"200","kind":"VM","logical_id":"vm200"},{"archive_size_bytes":3192,"id":"filebrowser","kind":"APP","logical_id":"app-filebrowser"},{"archive_size_bytes":153389,"id":"skladchik-reports-monitor","kind":"APP","logical_id":"app-skladchik-reports-monitor"},{"archive_size_bytes":270193550,"id":"immich","kind":"APP","logical_id":"app-immich"},{"archive_size_bytes":75379068,"id":"hbarhub","kind":"APP","logical_id":"app-hbarhub"},{"archive_size_bytes":8959200,"id":"control-bootstrap-v2","kind":"CONTROL","logical_id":"control-bootstrap-v2"}]} -{"clouds":[{"free_bytes":755855450788,"remote":"pve01-mail-01-crypt","total_bytes":1099511627776,"used_bytes":343656176988},{"free_bytes":755855386929,"remote":"pve01-mail-02-crypt","total_bytes":1099511627776,"used_bytes":343656240847}],"observed_at_epoch":1788460613,"schema":"homelab-backup-observability-snapshot-v1","targets":[{"archive_size_bytes":476345840,"id":"110","kind":"VM","logical_id":"vm110"},{"archive_size_bytes":369191552,"id":"111","kind":"VM","logical_id":"vm111"},{"archive_size_bytes":223627414,"id":"112","kind":"VM","logical_id":"vm112"},{"archive_size_bytes":221521137,"id":"113","kind":"VM","logical_id":"vm113"},{"archive_size_bytes":10611899437,"id":"130","kind":"VM","logical_id":"vm130"},{"archive_size_bytes":892033158,"id":"150","kind":"VM","logical_id":"vm150"},{"archive_size_bytes":3469023634,"id":"160","kind":"VM","logical_id":"vm160"},{"archive_size_bytes":7937058725,"id":"170","kind":"VM","logical_id":"vm170"},{"archive_size_bytes":2026047659,"id":"171","kind":"VM","logical_id":"vm171"},{"archive_size_bytes":1171789442,"id":"180","kind":"VM","logical_id":"vm180"},{"archive_size_bytes":1557482946,"id":"190","kind":"VM","logical_id":"vm190"},{"archive_size_bytes":386825665,"id":"200","kind":"VM","logical_id":"vm200"},{"archive_size_bytes":3192,"id":"filebrowser","kind":"APP","logical_id":"app-filebrowser"},{"archive_size_bytes":153389,"id":"skladchik-reports-monitor","kind":"APP","logical_id":"app-skladchik-reports-monitor"},{"archive_size_bytes":270193550,"id":"immich","kind":"APP","logical_id":"app-immich"},{"archive_size_bytes":75379068,"id":"hbarhub","kind":"APP","logical_id":"app-hbarhub"},{"archive_size_bytes":8959200,"id":"control-bootstrap-v2","kind":"CONTROL","logical_id":"control-bootstrap-v2"}]} -{"clouds":[{"free_bytes":751486275845,"remote":"pve01-mail-01-crypt","total_bytes":1099511627776,"used_bytes":348025351931},{"free_bytes":751486211986,"remote":"pve01-mail-02-crypt","total_bytes":1099511627776,"used_bytes":348025415790}],"observed_at_epoch":1788465932,"schema":"homelab-backup-observability-snapshot-v1","targets":[{"archive_size_bytes":476345840,"id":"110","kind":"VM","logical_id":"vm110"},{"archive_size_bytes":369892675,"id":"111","kind":"VM","logical_id":"vm111"},{"archive_size_bytes":223672749,"id":"112","kind":"VM","logical_id":"vm112"},{"archive_size_bytes":221529138,"id":"113","kind":"VM","logical_id":"vm113"},{"archive_size_bytes":10611899437,"id":"130","kind":"VM","logical_id":"vm130"},{"archive_size_bytes":892033158,"id":"150","kind":"VM","logical_id":"vm150"},{"archive_size_bytes":3553006761,"id":"160","kind":"VM","logical_id":"vm160"},{"archive_size_bytes":7937058725,"id":"170","kind":"VM","logical_id":"vm170"},{"archive_size_bytes":2026047659,"id":"171","kind":"VM","logical_id":"vm171"},{"archive_size_bytes":1171789442,"id":"180","kind":"VM","logical_id":"vm180"},{"archive_size_bytes":1557482946,"id":"190","kind":"VM","logical_id":"vm190"},{"archive_size_bytes":10998362942,"id":"9130","kind":"VM","logical_id":"vm9130"},{"archive_size_bytes":386825665,"id":"200","kind":"VM","logical_id":"vm200"},{"archive_size_bytes":3192,"id":"filebrowser","kind":"APP","logical_id":"app-filebrowser"},{"archive_size_bytes":153389,"id":"skladchik-reports-monitor","kind":"APP","logical_id":"app-skladchik-reports-monitor"},{"archive_size_bytes":270193550,"id":"immich","kind":"APP","logical_id":"app-immich"},{"archive_size_bytes":75379068,"id":"hbarhub","kind":"APP","logical_id":"app-hbarhub"},{"archive_size_bytes":8959200,"id":"control-bootstrap-v2","kind":"CONTROL","logical_id":"control-bootstrap-v2"}]} -{"clouds":[{"free_bytes":751477313023,"remote":"pve01-mail-01-crypt","total_bytes":1099511627776,"used_bytes":348034314753},{"free_bytes":751477249164,"remote":"pve01-mail-02-crypt","total_bytes":1099511627776,"used_bytes":348034378612}],"observed_at_epoch":1788469583,"schema":"homelab-backup-observability-snapshot-v1","targets":[{"archive_size_bytes":476345840,"id":"110","kind":"VM","logical_id":"vm110"},{"archive_size_bytes":369892675,"id":"111","kind":"VM","logical_id":"vm111"},{"archive_size_bytes":223672749,"id":"112","kind":"VM","logical_id":"vm112"},{"archive_size_bytes":221529138,"id":"113","kind":"VM","logical_id":"vm113"},{"archive_size_bytes":10611899437,"id":"130","kind":"VM","logical_id":"vm130"},{"archive_size_bytes":892033158,"id":"150","kind":"VM","logical_id":"vm150"},{"archive_size_bytes":3553006761,"id":"160","kind":"VM","logical_id":"vm160"},{"archive_size_bytes":7937058725,"id":"170","kind":"VM","logical_id":"vm170"},{"archive_size_bytes":2026047659,"id":"171","kind":"VM","logical_id":"vm171"},{"archive_size_bytes":1171789442,"id":"180","kind":"VM","logical_id":"vm180"},{"archive_size_bytes":1557482946,"id":"190","kind":"VM","logical_id":"vm190"},{"archive_size_bytes":10998362942,"id":"9130","kind":"VM","logical_id":"vm9130"},{"archive_size_bytes":386825665,"id":"200","kind":"VM","logical_id":"vm200"},{"archive_size_bytes":3192,"id":"filebrowser","kind":"APP","logical_id":"app-filebrowser"},{"archive_size_bytes":153389,"id":"skladchik-reports-monitor","kind":"APP","logical_id":"app-skladchik-reports-monitor"},{"archive_size_bytes":270193550,"id":"immich","kind":"APP","logical_id":"app-immich"},{"archive_size_bytes":75379068,"id":"hbarhub","kind":"APP","logical_id":"app-hbarhub"},{"archive_size_bytes":8959291,"id":"control-bootstrap-v2","kind":"CONTROL","logical_id":"control-bootstrap-v2"}]} -{"clouds":[{"free_bytes":751477313023,"remote":"pve01-mail-01-crypt","total_bytes":1099511627776,"used_bytes":348034314753},{"free_bytes":751477249164,"remote":"pve01-mail-02-crypt","total_bytes":1099511627776,"used_bytes":348034378612}],"observed_at_epoch":1788474989,"schema":"homelab-backup-observability-snapshot-v1","targets":[{"archive_size_bytes":476345840,"id":"110","kind":"VM","logical_id":"vm110"},{"archive_size_bytes":369892675,"id":"111","kind":"VM","logical_id":"vm111"},{"archive_size_bytes":223672749,"id":"112","kind":"VM","logical_id":"vm112"},{"archive_size_bytes":221529138,"id":"113","kind":"VM","logical_id":"vm113"},{"archive_size_bytes":10611899437,"id":"130","kind":"VM","logical_id":"vm130"},{"archive_size_bytes":892033158,"id":"150","kind":"VM","logical_id":"vm150"},{"archive_size_bytes":3553006761,"id":"160","kind":"VM","logical_id":"vm160"},{"archive_size_bytes":7937058725,"id":"170","kind":"VM","logical_id":"vm170"},{"archive_size_bytes":2026047659,"id":"171","kind":"VM","logical_id":"vm171"},{"archive_size_bytes":1171789442,"id":"180","kind":"VM","logical_id":"vm180"},{"archive_size_bytes":1557482946,"id":"190","kind":"VM","logical_id":"vm190"},{"archive_size_bytes":10998362942,"id":"9130","kind":"VM","logical_id":"vm9130"},{"archive_size_bytes":386825665,"id":"200","kind":"VM","logical_id":"vm200"},{"archive_size_bytes":3192,"id":"filebrowser","kind":"APP","logical_id":"app-filebrowser"},{"archive_size_bytes":153389,"id":"skladchik-reports-monitor","kind":"APP","logical_id":"app-skladchik-reports-monitor"},{"archive_size_bytes":270193550,"id":"immich","kind":"APP","logical_id":"app-immich"},{"archive_size_bytes":75379068,"id":"hbarhub","kind":"APP","logical_id":"app-hbarhub"},{"archive_size_bytes":8959291,"id":"control-bootstrap-v2","kind":"CONTROL","logical_id":"control-bootstrap-v2"}]} -{"clouds":[{"free_bytes":751477313023,"remote":"pve01-mail-01-crypt","total_bytes":1099511627776,"used_bytes":348034314753},{"free_bytes":751477249164,"remote":"pve01-mail-02-crypt","total_bytes":1099511627776,"used_bytes":348034378612}],"observed_at_epoch":1788478603,"schema":"homelab-backup-observability-snapshot-v1","targets":[{"archive_size_bytes":476345840,"id":"110","kind":"VM","logical_id":"vm110"},{"archive_size_bytes":369892675,"id":"111","kind":"VM","logical_id":"vm111"},{"archive_size_bytes":223672749,"id":"112","kind":"VM","logical_id":"vm112"},{"archive_size_bytes":221529138,"id":"113","kind":"VM","logical_id":"vm113"},{"archive_size_bytes":10611899437,"id":"130","kind":"VM","logical_id":"vm130"},{"archive_size_bytes":892033158,"id":"150","kind":"VM","logical_id":"vm150"},{"archive_size_bytes":3553006761,"id":"160","kind":"VM","logical_id":"vm160"},{"archive_size_bytes":7937058725,"id":"170","kind":"VM","logical_id":"vm170"},{"archive_size_bytes":2026047659,"id":"171","kind":"VM","logical_id":"vm171"},{"archive_size_bytes":1171789442,"id":"180","kind":"VM","logical_id":"vm180"},{"archive_size_bytes":1557482946,"id":"190","kind":"VM","logical_id":"vm190"},{"archive_size_bytes":10998362942,"id":"9130","kind":"VM","logical_id":"vm9130"},{"archive_size_bytes":386825665,"id":"200","kind":"VM","logical_id":"vm200"},{"archive_size_bytes":3192,"id":"filebrowser","kind":"APP","logical_id":"app-filebrowser"},{"archive_size_bytes":153389,"id":"skladchik-reports-monitor","kind":"APP","logical_id":"app-skladchik-reports-monitor"},{"archive_size_bytes":270193550,"id":"immich","kind":"APP","logical_id":"app-immich"},{"archive_size_bytes":75379068,"id":"hbarhub","kind":"APP","logical_id":"app-hbarhub"},{"archive_size_bytes":8959291,"id":"control-bootstrap-v2","kind":"CONTROL","logical_id":"control-bootstrap-v2"}]} -{"clouds":[{"free_bytes":751477313023,"remote":"pve01-mail-01-crypt","total_bytes":1099511627776,"used_bytes":348034314753},{"free_bytes":751477249164,"remote":"pve01-mail-02-crypt","total_bytes":1099511627776,"used_bytes":348034378612}],"observed_at_epoch":1788482209,"schema":"homelab-backup-observability-snapshot-v1","targets":[{"archive_size_bytes":476345840,"id":"110","kind":"VM","logical_id":"vm110"},{"archive_size_bytes":369892675,"id":"111","kind":"VM","logical_id":"vm111"},{"archive_size_bytes":223672749,"id":"112","kind":"VM","logical_id":"vm112"},{"archive_size_bytes":221529138,"id":"113","kind":"VM","logical_id":"vm113"},{"archive_size_bytes":10611899437,"id":"130","kind":"VM","logical_id":"vm130"},{"archive_size_bytes":892033158,"id":"150","kind":"VM","logical_id":"vm150"},{"archive_size_bytes":3553006761,"id":"160","kind":"VM","logical_id":"vm160"},{"archive_size_bytes":7937058725,"id":"170","kind":"VM","logical_id":"vm170"},{"archive_size_bytes":2026047659,"id":"171","kind":"VM","logical_id":"vm171"},{"archive_size_bytes":1171789442,"id":"180","kind":"VM","logical_id":"vm180"},{"archive_size_bytes":1557482946,"id":"190","kind":"VM","logical_id":"vm190"},{"archive_size_bytes":10998362942,"id":"9130","kind":"VM","logical_id":"vm9130"},{"archive_size_bytes":386825665,"id":"200","kind":"VM","logical_id":"vm200"},{"archive_size_bytes":3192,"id":"filebrowser","kind":"APP","logical_id":"app-filebrowser"},{"archive_size_bytes":153389,"id":"skladchik-reports-monitor","kind":"APP","logical_id":"app-skladchik-reports-monitor"},{"archive_size_bytes":270193550,"id":"immich","kind":"APP","logical_id":"app-immich"},{"archive_size_bytes":75379068,"id":"hbarhub","kind":"APP","logical_id":"app-hbarhub"},{"archive_size_bytes":8959291,"id":"control-bootstrap-v2","kind":"CONTROL","logical_id":"control-bootstrap-v2"}]} -{"clouds":[{"free_bytes":751477313023,"remote":"pve01-mail-01-crypt","total_bytes":1099511627776,"used_bytes":348034314753},{"free_bytes":751477249164,"remote":"pve01-mail-02-crypt","total_bytes":1099511627776,"used_bytes":348034378612}],"observed_at_epoch":1788487523,"schema":"homelab-backup-observability-snapshot-v1","targets":[{"archive_size_bytes":476345840,"id":"110","kind":"VM","logical_id":"vm110"},{"archive_size_bytes":369892675,"id":"111","kind":"VM","logical_id":"vm111"},{"archive_size_bytes":223672749,"id":"112","kind":"VM","logical_id":"vm112"},{"archive_size_bytes":221529138,"id":"113","kind":"VM","logical_id":"vm113"},{"archive_size_bytes":10611899437,"id":"130","kind":"VM","logical_id":"vm130"},{"archive_size_bytes":892033158,"id":"150","kind":"VM","logical_id":"vm150"},{"archive_size_bytes":3553006761,"id":"160","kind":"VM","logical_id":"vm160"},{"archive_size_bytes":7937058725,"id":"170","kind":"VM","logical_id":"vm170"},{"archive_size_bytes":2026047659,"id":"171","kind":"VM","logical_id":"vm171"},{"archive_size_bytes":1171789442,"id":"180","kind":"VM","logical_id":"vm180"},{"archive_size_bytes":1557482946,"id":"190","kind":"VM","logical_id":"vm190"},{"archive_size_bytes":10998362942,"id":"9130","kind":"VM","logical_id":"vm9130"},{"archive_size_bytes":386825665,"id":"200","kind":"VM","logical_id":"vm200"},{"archive_size_bytes":3192,"id":"filebrowser","kind":"APP","logical_id":"app-filebrowser"},{"archive_size_bytes":153389,"id":"skladchik-reports-monitor","kind":"APP","logical_id":"app-skladchik-reports-monitor"},{"archive_size_bytes":270193550,"id":"immich","kind":"APP","logical_id":"app-immich"},{"archive_size_bytes":75379068,"id":"hbarhub","kind":"APP","logical_id":"app-hbarhub"},{"archive_size_bytes":8959291,"id":"control-bootstrap-v2","kind":"CONTROL","logical_id":"control-bootstrap-v2"}]} -{"clouds":[{"free_bytes":751477313023,"remote":"pve01-mail-01-crypt","total_bytes":1099511627776,"used_bytes":348034314753},{"free_bytes":751477249164,"remote":"pve01-mail-02-crypt","total_bytes":1099511627776,"used_bytes":348034378612}],"observed_at_epoch":1788491188,"schema":"homelab-backup-observability-snapshot-v1","targets":[{"archive_size_bytes":476345840,"id":"110","kind":"VM","logical_id":"vm110"},{"archive_size_bytes":369892675,"id":"111","kind":"VM","logical_id":"vm111"},{"archive_size_bytes":223672749,"id":"112","kind":"VM","logical_id":"vm112"},{"archive_size_bytes":221529138,"id":"113","kind":"VM","logical_id":"vm113"},{"archive_size_bytes":10611899437,"id":"130","kind":"VM","logical_id":"vm130"},{"archive_size_bytes":892033158,"id":"150","kind":"VM","logical_id":"vm150"},{"archive_size_bytes":3553006761,"id":"160","kind":"VM","logical_id":"vm160"},{"archive_size_bytes":7937058725,"id":"170","kind":"VM","logical_id":"vm170"},{"archive_size_bytes":2026047659,"id":"171","kind":"VM","logical_id":"vm171"},{"archive_size_bytes":1171789442,"id":"180","kind":"VM","logical_id":"vm180"},{"archive_size_bytes":1557482946,"id":"190","kind":"VM","logical_id":"vm190"},{"archive_size_bytes":10998362942,"id":"9130","kind":"VM","logical_id":"vm9130"},{"archive_size_bytes":386825665,"id":"200","kind":"VM","logical_id":"vm200"},{"archive_size_bytes":3192,"id":"filebrowser","kind":"APP","logical_id":"app-filebrowser"},{"archive_size_bytes":153389,"id":"skladchik-reports-monitor","kind":"APP","logical_id":"app-skladchik-reports-monitor"},{"archive_size_bytes":270193550,"id":"immich","kind":"APP","logical_id":"app-immich"},{"archive_size_bytes":75379068,"id":"hbarhub","kind":"APP","logical_id":"app-hbarhub"},{"archive_size_bytes":8959291,"id":"control-bootstrap-v2","kind":"CONTROL","logical_id":"control-bootstrap-v2"}]} -{"clouds":[{"free_bytes":835463704954,"remote":"pve01-mail-01-crypt","total_bytes":1099511627776,"used_bytes":264047922822},{"free_bytes":835463641095,"remote":"pve01-mail-02-crypt","total_bytes":1099511627776,"used_bytes":264047986681}],"observed_at_epoch":1788496558,"schema":"homelab-backup-observability-snapshot-v1","targets":[{"archive_size_bytes":476345840,"id":"110","kind":"VM","logical_id":"vm110"},{"archive_size_bytes":369892675,"id":"111","kind":"VM","logical_id":"vm111"},{"archive_size_bytes":223672749,"id":"112","kind":"VM","logical_id":"vm112"},{"archive_size_bytes":221529138,"id":"113","kind":"VM","logical_id":"vm113"},{"archive_size_bytes":10611899437,"id":"130","kind":"VM","logical_id":"vm130"},{"archive_size_bytes":892033158,"id":"150","kind":"VM","logical_id":"vm150"},{"archive_size_bytes":3553006761,"id":"160","kind":"VM","logical_id":"vm160"},{"archive_size_bytes":7937058725,"id":"170","kind":"VM","logical_id":"vm170"},{"archive_size_bytes":2026047659,"id":"171","kind":"VM","logical_id":"vm171"},{"archive_size_bytes":1171789442,"id":"180","kind":"VM","logical_id":"vm180"},{"archive_size_bytes":1557482946,"id":"190","kind":"VM","logical_id":"vm190"},{"archive_size_bytes":10998362942,"id":"9130","kind":"VM","logical_id":"vm9130"},{"archive_size_bytes":386825665,"id":"200","kind":"VM","logical_id":"vm200"},{"archive_size_bytes":3192,"id":"filebrowser","kind":"APP","logical_id":"app-filebrowser"},{"archive_size_bytes":153389,"id":"skladchik-reports-monitor","kind":"APP","logical_id":"app-skladchik-reports-monitor"},{"archive_size_bytes":270193550,"id":"immich","kind":"APP","logical_id":"app-immich"},{"archive_size_bytes":75379068,"id":"hbarhub","kind":"APP","logical_id":"app-hbarhub"},{"archive_size_bytes":8959291,"id":"control-bootstrap-v2","kind":"CONTROL","logical_id":"control-bootstrap-v2"}]} -{"clouds":[{"free_bytes":815114087404,"remote":"pve01-mail-01-crypt","total_bytes":1099511627776,"used_bytes":284397540372},{"free_bytes":815114023545,"remote":"pve01-mail-02-crypt","total_bytes":1099511627776,"used_bytes":284397604231}],"observed_at_epoch":1788503153,"schema":"homelab-backup-observability-snapshot-v1","targets":[{"archive_size_bytes":478357303,"id":"110","kind":"VM","logical_id":"vm110"},{"archive_size_bytes":369892675,"id":"111","kind":"VM","logical_id":"vm111"},{"archive_size_bytes":223672749,"id":"112","kind":"VM","logical_id":"vm112"},{"archive_size_bytes":221529138,"id":"113","kind":"VM","logical_id":"vm113"},{"archive_size_bytes":887338816,"id":"150","kind":"VM","logical_id":"vm150"},{"archive_size_bytes":3553006761,"id":"160","kind":"VM","logical_id":"vm160"},{"archive_size_bytes":7974616162,"id":"170","kind":"VM","logical_id":"vm170"},{"archive_size_bytes":2026047659,"id":"171","kind":"VM","logical_id":"vm171"},{"archive_size_bytes":1557482946,"id":"190","kind":"VM","logical_id":"vm190"},{"archive_size_bytes":10998362942,"id":"9130","kind":"VM","logical_id":"vm9130"},{"archive_size_bytes":3192,"id":"filebrowser","kind":"APP","logical_id":"app-filebrowser"},{"archive_size_bytes":153389,"id":"skladchik-reports-monitor","kind":"APP","logical_id":"app-skladchik-reports-monitor"},{"archive_size_bytes":270193550,"id":"immich","kind":"APP","logical_id":"app-immich"},{"archive_size_bytes":75379068,"id":"hbarhub","kind":"APP","logical_id":"app-hbarhub"},{"archive_size_bytes":8959291,"id":"control-bootstrap-v2","kind":"CONTROL","logical_id":"control-bootstrap-v2"}]} -{"clouds":[{"free_bytes":812739438805,"remote":"pve01-mail-01-crypt","total_bytes":1099511627776,"used_bytes":286772188971},{"free_bytes":812739374946,"remote":"pve01-mail-02-crypt","total_bytes":1099511627776,"used_bytes":286772252830}],"observed_at_epoch":1788507354,"schema":"homelab-backup-observability-snapshot-v1","targets":[{"archive_size_bytes":478357303,"id":"110","kind":"VM","logical_id":"vm110"},{"archive_size_bytes":369892675,"id":"111","kind":"VM","logical_id":"vm111"},{"archive_size_bytes":223672749,"id":"112","kind":"VM","logical_id":"vm112"},{"archive_size_bytes":221529138,"id":"113","kind":"VM","logical_id":"vm113"},{"archive_size_bytes":10620176786,"id":"130","kind":"VM","logical_id":"vm130"},{"archive_size_bytes":887338816,"id":"150","kind":"VM","logical_id":"vm150"},{"archive_size_bytes":3553006761,"id":"160","kind":"VM","logical_id":"vm160"},{"archive_size_bytes":7974616162,"id":"170","kind":"VM","logical_id":"vm170"},{"archive_size_bytes":2028255985,"id":"171","kind":"VM","logical_id":"vm171"},{"archive_size_bytes":1557482946,"id":"190","kind":"VM","logical_id":"vm190"},{"archive_size_bytes":10998362942,"id":"9130","kind":"VM","logical_id":"vm9130"},{"archive_size_bytes":3197,"id":"filebrowser","kind":"APP","logical_id":"app-filebrowser"},{"archive_size_bytes":154716,"id":"skladchik-reports-monitor","kind":"APP","logical_id":"app-skladchik-reports-monitor"},{"archive_size_bytes":270193492,"id":"immich","kind":"APP","logical_id":"app-immich"},{"archive_size_bytes":75453412,"id":"hbarhub","kind":"APP","logical_id":"app-hbarhub"},{"archive_size_bytes":8959291,"id":"control-bootstrap-v2","kind":"CONTROL","logical_id":"control-bootstrap-v2"}]} -{"clouds":[{"free_bytes":809167318295,"remote":"pve01-mail-01-crypt","total_bytes":1099511627776,"used_bytes":290344309481},{"free_bytes":809167254436,"remote":"pve01-mail-02-crypt","total_bytes":1099511627776,"used_bytes":290344373340}],"observed_at_epoch":1788512752,"schema":"homelab-backup-observability-snapshot-v1","targets":[{"archive_size_bytes":478357303,"id":"110","kind":"VM","logical_id":"vm110"},{"archive_size_bytes":369892675,"id":"111","kind":"VM","logical_id":"vm111"},{"archive_size_bytes":223672749,"id":"112","kind":"VM","logical_id":"vm112"},{"archive_size_bytes":221529138,"id":"113","kind":"VM","logical_id":"vm113"},{"archive_size_bytes":10620176786,"id":"130","kind":"VM","logical_id":"vm130"},{"archive_size_bytes":887338816,"id":"150","kind":"VM","logical_id":"vm150"},{"archive_size_bytes":3571246511,"id":"160","kind":"VM","logical_id":"vm160"},{"archive_size_bytes":7974616162,"id":"170","kind":"VM","logical_id":"vm170"},{"archive_size_bytes":2028255985,"id":"171","kind":"VM","logical_id":"vm171"},{"archive_size_bytes":1557482946,"id":"190","kind":"VM","logical_id":"vm190"},{"archive_size_bytes":10998362942,"id":"9130","kind":"VM","logical_id":"vm9130"},{"archive_size_bytes":3197,"id":"filebrowser","kind":"APP","logical_id":"app-filebrowser"},{"archive_size_bytes":154716,"id":"skladchik-reports-monitor","kind":"APP","logical_id":"app-skladchik-reports-monitor"},{"archive_size_bytes":270193492,"id":"immich","kind":"APP","logical_id":"app-immich"},{"archive_size_bytes":75453412,"id":"hbarhub","kind":"APP","logical_id":"app-hbarhub"},{"archive_size_bytes":8959291,"id":"control-bootstrap-v2","kind":"CONTROL","logical_id":"control-bootstrap-v2"}]} -{"clouds":[{"free_bytes":809158344080,"remote":"pve01-mail-01-crypt","total_bytes":1099511627776,"used_bytes":290353283696},{"free_bytes":809158280221,"remote":"pve01-mail-02-crypt","total_bytes":1099511627776,"used_bytes":290353347555}],"observed_at_epoch":1788516392,"schema":"homelab-backup-observability-snapshot-v1","targets":[{"archive_size_bytes":478357303,"id":"110","kind":"VM","logical_id":"vm110"},{"archive_size_bytes":369892675,"id":"111","kind":"VM","logical_id":"vm111"},{"archive_size_bytes":223672749,"id":"112","kind":"VM","logical_id":"vm112"},{"archive_size_bytes":221529138,"id":"113","kind":"VM","logical_id":"vm113"},{"archive_size_bytes":10620176786,"id":"130","kind":"VM","logical_id":"vm130"},{"archive_size_bytes":887338816,"id":"150","kind":"VM","logical_id":"vm150"},{"archive_size_bytes":3571246511,"id":"160","kind":"VM","logical_id":"vm160"},{"archive_size_bytes":7974616162,"id":"170","kind":"VM","logical_id":"vm170"},{"archive_size_bytes":2028255985,"id":"171","kind":"VM","logical_id":"vm171"},{"archive_size_bytes":1557482946,"id":"190","kind":"VM","logical_id":"vm190"},{"archive_size_bytes":10998362942,"id":"9130","kind":"VM","logical_id":"vm9130"},{"archive_size_bytes":3197,"id":"filebrowser","kind":"APP","logical_id":"app-filebrowser"},{"archive_size_bytes":154716,"id":"skladchik-reports-monitor","kind":"APP","logical_id":"app-skladchik-reports-monitor"},{"archive_size_bytes":270193492,"id":"immich","kind":"APP","logical_id":"app-immich"},{"archive_size_bytes":75453412,"id":"hbarhub","kind":"APP","logical_id":"app-hbarhub"},{"archive_size_bytes":8970684,"id":"control-bootstrap-v2","kind":"CONTROL","logical_id":"control-bootstrap-v2"}]} -{"clouds":[{"free_bytes":809158344080,"remote":"pve01-mail-01-crypt","total_bytes":1099511627776,"used_bytes":290353283696},{"free_bytes":809158280221,"remote":"pve01-mail-02-crypt","total_bytes":1099511627776,"used_bytes":290353347555}],"observed_at_epoch":1788520010,"schema":"homelab-backup-observability-snapshot-v1","targets":[{"archive_size_bytes":478357303,"id":"110","kind":"VM","logical_id":"vm110"},{"archive_size_bytes":369892675,"id":"111","kind":"VM","logical_id":"vm111"},{"archive_size_bytes":223672749,"id":"112","kind":"VM","logical_id":"vm112"},{"archive_size_bytes":221529138,"id":"113","kind":"VM","logical_id":"vm113"},{"archive_size_bytes":10620176786,"id":"130","kind":"VM","logical_id":"vm130"},{"archive_size_bytes":887338816,"id":"150","kind":"VM","logical_id":"vm150"},{"archive_size_bytes":3571246511,"id":"160","kind":"VM","logical_id":"vm160"},{"archive_size_bytes":7974616162,"id":"170","kind":"VM","logical_id":"vm170"},{"archive_size_bytes":2028255985,"id":"171","kind":"VM","logical_id":"vm171"},{"archive_size_bytes":1557482946,"id":"190","kind":"VM","logical_id":"vm190"},{"archive_size_bytes":10998362942,"id":"9130","kind":"VM","logical_id":"vm9130"},{"archive_size_bytes":3197,"id":"filebrowser","kind":"APP","logical_id":"app-filebrowser"},{"archive_size_bytes":154716,"id":"skladchik-reports-monitor","kind":"APP","logical_id":"app-skladchik-reports-monitor"},{"archive_size_bytes":270193492,"id":"immich","kind":"APP","logical_id":"app-immich"},{"archive_size_bytes":75453412,"id":"hbarhub","kind":"APP","logical_id":"app-hbarhub"},{"archive_size_bytes":8970684,"id":"control-bootstrap-v2","kind":"CONTROL","logical_id":"control-bootstrap-v2"}]} -{"clouds":[{"free_bytes":809158344080,"remote":"pve01-mail-01-crypt","total_bytes":1099511627776,"used_bytes":290353283696},{"free_bytes":809158280221,"remote":"pve01-mail-02-crypt","total_bytes":1099511627776,"used_bytes":290353347555}],"observed_at_epoch":1788525352,"schema":"homelab-backup-observability-snapshot-v1","targets":[{"archive_size_bytes":478357303,"id":"110","kind":"VM","logical_id":"vm110"},{"archive_size_bytes":369892675,"id":"111","kind":"VM","logical_id":"vm111"},{"archive_size_bytes":223672749,"id":"112","kind":"VM","logical_id":"vm112"},{"archive_size_bytes":221529138,"id":"113","kind":"VM","logical_id":"vm113"},{"archive_size_bytes":10620176786,"id":"130","kind":"VM","logical_id":"vm130"},{"archive_size_bytes":887338816,"id":"150","kind":"VM","logical_id":"vm150"},{"archive_size_bytes":3571246511,"id":"160","kind":"VM","logical_id":"vm160"},{"archive_size_bytes":7974616162,"id":"170","kind":"VM","logical_id":"vm170"},{"archive_size_bytes":2028255985,"id":"171","kind":"VM","logical_id":"vm171"},{"archive_size_bytes":1557482946,"id":"190","kind":"VM","logical_id":"vm190"},{"archive_size_bytes":10998362942,"id":"9130","kind":"VM","logical_id":"vm9130"},{"archive_size_bytes":3197,"id":"filebrowser","kind":"APP","logical_id":"app-filebrowser"},{"archive_size_bytes":154716,"id":"skladchik-reports-monitor","kind":"APP","logical_id":"app-skladchik-reports-monitor"},{"archive_size_bytes":270193492,"id":"immich","kind":"APP","logical_id":"app-immich"},{"archive_size_bytes":75453412,"id":"hbarhub","kind":"APP","logical_id":"app-hbarhub"},{"archive_size_bytes":8970684,"id":"control-bootstrap-v2","kind":"CONTROL","logical_id":"control-bootstrap-v2"}]} -{"clouds":[{"free_bytes":809158344080,"remote":"pve01-mail-01-crypt","total_bytes":1099511627776,"used_bytes":290353283696},{"free_bytes":809158280221,"remote":"pve01-mail-02-crypt","total_bytes":1099511627776,"used_bytes":290353347555}],"observed_at_epoch":1788528952,"schema":"homelab-backup-observability-snapshot-v1","targets":[{"archive_size_bytes":478357303,"id":"110","kind":"VM","logical_id":"vm110"},{"archive_size_bytes":369892675,"id":"111","kind":"VM","logical_id":"vm111"},{"archive_size_bytes":223672749,"id":"112","kind":"VM","logical_id":"vm112"},{"archive_size_bytes":221529138,"id":"113","kind":"VM","logical_id":"vm113"},{"archive_size_bytes":10620176786,"id":"130","kind":"VM","logical_id":"vm130"},{"archive_size_bytes":887338816,"id":"150","kind":"VM","logical_id":"vm150"},{"archive_size_bytes":3571246511,"id":"160","kind":"VM","logical_id":"vm160"},{"archive_size_bytes":7974616162,"id":"170","kind":"VM","logical_id":"vm170"},{"archive_size_bytes":2028255985,"id":"171","kind":"VM","logical_id":"vm171"},{"archive_size_bytes":1557482946,"id":"190","kind":"VM","logical_id":"vm190"},{"archive_size_bytes":10998362942,"id":"9130","kind":"VM","logical_id":"vm9130"},{"archive_size_bytes":3197,"id":"filebrowser","kind":"APP","logical_id":"app-filebrowser"},{"archive_size_bytes":154716,"id":"skladchik-reports-monitor","kind":"APP","logical_id":"app-skladchik-reports-monitor"},{"archive_size_bytes":270193492,"id":"immich","kind":"APP","logical_id":"app-immich"},{"archive_size_bytes":75453412,"id":"hbarhub","kind":"APP","logical_id":"app-hbarhub"},{"archive_size_bytes":8970684,"id":"control-bootstrap-v2","kind":"CONTROL","logical_id":"control-bootstrap-v2"}]} -{"clouds":[{"free_bytes":809158344080,"remote":"pve01-mail-01-crypt","total_bytes":1099511627776,"used_bytes":290353283696},{"free_bytes":809158280221,"remote":"pve01-mail-02-crypt","total_bytes":1099511627776,"used_bytes":290353347555}],"observed_at_epoch":1788532552,"schema":"homelab-backup-observability-snapshot-v1","targets":[{"archive_size_bytes":478357303,"id":"110","kind":"VM","logical_id":"vm110"},{"archive_size_bytes":369892675,"id":"111","kind":"VM","logical_id":"vm111"},{"archive_size_bytes":223672749,"id":"112","kind":"VM","logical_id":"vm112"},{"archive_size_bytes":221529138,"id":"113","kind":"VM","logical_id":"vm113"},{"archive_size_bytes":10620176786,"id":"130","kind":"VM","logical_id":"vm130"},{"archive_size_bytes":887338816,"id":"150","kind":"VM","logical_id":"vm150"},{"archive_size_bytes":3571246511,"id":"160","kind":"VM","logical_id":"vm160"},{"archive_size_bytes":7974616162,"id":"170","kind":"VM","logical_id":"vm170"},{"archive_size_bytes":2028255985,"id":"171","kind":"VM","logical_id":"vm171"},{"archive_size_bytes":1557482946,"id":"190","kind":"VM","logical_id":"vm190"},{"archive_size_bytes":10998362942,"id":"9130","kind":"VM","logical_id":"vm9130"},{"archive_size_bytes":3197,"id":"filebrowser","kind":"APP","logical_id":"app-filebrowser"},{"archive_size_bytes":154716,"id":"skladchik-reports-monitor","kind":"APP","logical_id":"app-skladchik-reports-monitor"},{"archive_size_bytes":270193492,"id":"immich","kind":"APP","logical_id":"app-immich"},{"archive_size_bytes":75453412,"id":"hbarhub","kind":"APP","logical_id":"app-hbarhub"},{"archive_size_bytes":8970684,"id":"control-bootstrap-v2","kind":"CONTROL","logical_id":"control-bootstrap-v2"}]} -{"clouds":[{"free_bytes":809158344080,"remote":"pve01-mail-01-crypt","total_bytes":1099511627776,"used_bytes":290353283696},{"free_bytes":809158280221,"remote":"pve01-mail-02-crypt","total_bytes":1099511627776,"used_bytes":290353347555}],"observed_at_epoch":1788536192,"schema":"homelab-backup-observability-snapshot-v1","targets":[{"archive_size_bytes":478357303,"id":"110","kind":"VM","logical_id":"vm110"},{"archive_size_bytes":369892675,"id":"111","kind":"VM","logical_id":"vm111"},{"archive_size_bytes":223672749,"id":"112","kind":"VM","logical_id":"vm112"},{"archive_size_bytes":221529138,"id":"113","kind":"VM","logical_id":"vm113"},{"archive_size_bytes":10620176786,"id":"130","kind":"VM","logical_id":"vm130"},{"archive_size_bytes":887338816,"id":"150","kind":"VM","logical_id":"vm150"},{"archive_size_bytes":3571246511,"id":"160","kind":"VM","logical_id":"vm160"},{"archive_size_bytes":7974616162,"id":"170","kind":"VM","logical_id":"vm170"},{"archive_size_bytes":2028255985,"id":"171","kind":"VM","logical_id":"vm171"},{"archive_size_bytes":1557482946,"id":"190","kind":"VM","logical_id":"vm190"},{"archive_size_bytes":10998362942,"id":"9130","kind":"VM","logical_id":"vm9130"},{"archive_size_bytes":3197,"id":"filebrowser","kind":"APP","logical_id":"app-filebrowser"},{"archive_size_bytes":154716,"id":"skladchik-reports-monitor","kind":"APP","logical_id":"app-skladchik-reports-monitor"},{"archive_size_bytes":270193492,"id":"immich","kind":"APP","logical_id":"app-immich"},{"archive_size_bytes":75453412,"id":"hbarhub","kind":"APP","logical_id":"app-hbarhub"},{"archive_size_bytes":8970684,"id":"control-bootstrap-v2","kind":"CONTROL","logical_id":"control-bootstrap-v2"}]} -{"clouds":[{"free_bytes":809158344080,"remote":"pve01-mail-01-crypt","total_bytes":1099511627776,"used_bytes":290353283696},{"free_bytes":809158280221,"remote":"pve01-mail-02-crypt","total_bytes":1099511627776,"used_bytes":290353347555}],"observed_at_epoch":1788541532,"schema":"homelab-backup-observability-snapshot-v1","targets":[{"archive_size_bytes":478357303,"id":"110","kind":"VM","logical_id":"vm110"},{"archive_size_bytes":369892675,"id":"111","kind":"VM","logical_id":"vm111"},{"archive_size_bytes":223672749,"id":"112","kind":"VM","logical_id":"vm112"},{"archive_size_bytes":221529138,"id":"113","kind":"VM","logical_id":"vm113"},{"archive_size_bytes":10620176786,"id":"130","kind":"VM","logical_id":"vm130"},{"archive_size_bytes":887338816,"id":"150","kind":"VM","logical_id":"vm150"},{"archive_size_bytes":3571246511,"id":"160","kind":"VM","logical_id":"vm160"},{"archive_size_bytes":7974616162,"id":"170","kind":"VM","logical_id":"vm170"},{"archive_size_bytes":2028255985,"id":"171","kind":"VM","logical_id":"vm171"},{"archive_size_bytes":1557482946,"id":"190","kind":"VM","logical_id":"vm190"},{"archive_size_bytes":10998362942,"id":"9130","kind":"VM","logical_id":"vm9130"},{"archive_size_bytes":3197,"id":"filebrowser","kind":"APP","logical_id":"app-filebrowser"},{"archive_size_bytes":154716,"id":"skladchik-reports-monitor","kind":"APP","logical_id":"app-skladchik-reports-monitor"},{"archive_size_bytes":270193492,"id":"immich","kind":"APP","logical_id":"app-immich"},{"archive_size_bytes":75453412,"id":"hbarhub","kind":"APP","logical_id":"app-hbarhub"},{"archive_size_bytes":8970684,"id":"control-bootstrap-v2","kind":"CONTROL","logical_id":"control-bootstrap-v2"}]} -{"clouds":[{"free_bytes":809158344080,"remote":"pve01-mail-01-crypt","total_bytes":1099511627776,"used_bytes":290353283696},{"free_bytes":809158280221,"remote":"pve01-mail-02-crypt","total_bytes":1099511627776,"used_bytes":290353347555}],"observed_at_epoch":1788545210,"schema":"homelab-backup-observability-snapshot-v1","targets":[{"archive_size_bytes":478357303,"id":"110","kind":"VM","logical_id":"vm110"},{"archive_size_bytes":369892675,"id":"111","kind":"VM","logical_id":"vm111"},{"archive_size_bytes":223672749,"id":"112","kind":"VM","logical_id":"vm112"},{"archive_size_bytes":221529138,"id":"113","kind":"VM","logical_id":"vm113"},{"archive_size_bytes":10620176786,"id":"130","kind":"VM","logical_id":"vm130"},{"archive_size_bytes":887338816,"id":"150","kind":"VM","logical_id":"vm150"},{"archive_size_bytes":3571246511,"id":"160","kind":"VM","logical_id":"vm160"},{"archive_size_bytes":7974616162,"id":"170","kind":"VM","logical_id":"vm170"},{"archive_size_bytes":2028255985,"id":"171","kind":"VM","logical_id":"vm171"},{"archive_size_bytes":1557482946,"id":"190","kind":"VM","logical_id":"vm190"},{"archive_size_bytes":10998362942,"id":"9130","kind":"VM","logical_id":"vm9130"},{"archive_size_bytes":3197,"id":"filebrowser","kind":"APP","logical_id":"app-filebrowser"},{"archive_size_bytes":154716,"id":"skladchik-reports-monitor","kind":"APP","logical_id":"app-skladchik-reports-monitor"},{"archive_size_bytes":270193492,"id":"immich","kind":"APP","logical_id":"app-immich"},{"archive_size_bytes":75453412,"id":"hbarhub","kind":"APP","logical_id":"app-hbarhub"},{"archive_size_bytes":8970684,"id":"control-bootstrap-v2","kind":"CONTROL","logical_id":"control-bootstrap-v2"}]} -{"clouds":[{"free_bytes":789729590488,"remote":"pve01-mail-01-crypt","total_bytes":1099511627776,"used_bytes":309782037288},{"free_bytes":789729526629,"remote":"pve01-mail-02-crypt","total_bytes":1099511627776,"used_bytes":309782101147}],"observed_at_epoch":1788550592,"schema":"homelab-backup-observability-snapshot-v1","targets":[{"archive_size_bytes":478357303,"id":"110","kind":"VM","logical_id":"vm110"},{"archive_size_bytes":369892675,"id":"111","kind":"VM","logical_id":"vm111"},{"archive_size_bytes":223672749,"id":"112","kind":"VM","logical_id":"vm112"},{"archive_size_bytes":221529138,"id":"113","kind":"VM","logical_id":"vm113"},{"archive_size_bytes":10614525791,"id":"130","kind":"VM","logical_id":"vm130"},{"archive_size_bytes":887338816,"id":"150","kind":"VM","logical_id":"vm150"},{"archive_size_bytes":3571246511,"id":"160","kind":"VM","logical_id":"vm160"},{"archive_size_bytes":6440615330,"id":"170","kind":"VM","logical_id":"vm170"},{"archive_size_bytes":1557482946,"id":"190","kind":"VM","logical_id":"vm190"},{"archive_size_bytes":10998362942,"id":"9130","kind":"VM","logical_id":"vm9130"},{"archive_size_bytes":3182,"id":"filebrowser","kind":"APP","logical_id":"app-filebrowser"},{"archive_size_bytes":156203,"id":"skladchik-reports-monitor","kind":"APP","logical_id":"app-skladchik-reports-monitor"},{"archive_size_bytes":270193546,"id":"immich","kind":"APP","logical_id":"app-immich"},{"archive_size_bytes":75522547,"id":"hbarhub","kind":"APP","logical_id":"app-hbarhub"},{"archive_size_bytes":8970684,"id":"control-bootstrap-v2","kind":"CONTROL","logical_id":"control-bootstrap-v2"}]} -{"clouds":[{"free_bytes":789729589363,"remote":"pve01-mail-01-crypt","total_bytes":1099511627776,"used_bytes":309782038413},{"free_bytes":789729525504,"remote":"pve01-mail-02-crypt","total_bytes":1099511627776,"used_bytes":309782102272}],"observed_at_epoch":1788554210,"schema":"homelab-backup-observability-snapshot-v1","targets":[{"archive_size_bytes":478357303,"id":"110","kind":"VM","logical_id":"vm110"},{"archive_size_bytes":369892675,"id":"111","kind":"VM","logical_id":"vm111"},{"archive_size_bytes":223672749,"id":"112","kind":"VM","logical_id":"vm112"},{"archive_size_bytes":221529138,"id":"113","kind":"VM","logical_id":"vm113"},{"archive_size_bytes":10614525791,"id":"130","kind":"VM","logical_id":"vm130"},{"archive_size_bytes":887338816,"id":"150","kind":"VM","logical_id":"vm150"},{"archive_size_bytes":3571246511,"id":"160","kind":"VM","logical_id":"vm160"},{"archive_size_bytes":6440615330,"id":"170","kind":"VM","logical_id":"vm170"},{"archive_size_bytes":2022982219,"id":"171","kind":"VM","logical_id":"vm171"},{"archive_size_bytes":1557482946,"id":"190","kind":"VM","logical_id":"vm190"},{"archive_size_bytes":10998362942,"id":"9130","kind":"VM","logical_id":"vm9130"},{"archive_size_bytes":3182,"id":"filebrowser","kind":"APP","logical_id":"app-filebrowser"},{"archive_size_bytes":156203,"id":"skladchik-reports-monitor","kind":"APP","logical_id":"app-skladchik-reports-monitor"},{"archive_size_bytes":270193546,"id":"immich","kind":"APP","logical_id":"app-immich"},{"archive_size_bytes":75522547,"id":"hbarhub","kind":"APP","logical_id":"app-hbarhub"},{"archive_size_bytes":8970684,"id":"control-bootstrap-v2","kind":"CONTROL","logical_id":"control-bootstrap-v2"}]} -{"clouds":[{"free_bytes":786104095977,"remote":"pve01-mail-01-crypt","total_bytes":1099511627776,"used_bytes":313407531799},{"free_bytes":786113009899,"remote":"pve01-mail-02-crypt","total_bytes":1099511627776,"used_bytes":313398617877}],"observed_at_epoch":1788559610,"schema":"homelab-backup-observability-snapshot-v1","targets":[{"archive_size_bytes":478357303,"id":"110","kind":"VM","logical_id":"vm110"},{"archive_size_bytes":369892675,"id":"111","kind":"VM","logical_id":"vm111"},{"archive_size_bytes":223672749,"id":"112","kind":"VM","logical_id":"vm112"},{"archive_size_bytes":221529138,"id":"113","kind":"VM","logical_id":"vm113"},{"archive_size_bytes":10614525791,"id":"130","kind":"VM","logical_id":"vm130"},{"archive_size_bytes":887338816,"id":"150","kind":"VM","logical_id":"vm150"},{"archive_size_bytes":3615630758,"id":"160","kind":"VM","logical_id":"vm160"},{"archive_size_bytes":6440615330,"id":"170","kind":"VM","logical_id":"vm170"},{"archive_size_bytes":2022982219,"id":"171","kind":"VM","logical_id":"vm171"},{"archive_size_bytes":1557482946,"id":"190","kind":"VM","logical_id":"vm190"},{"archive_size_bytes":10998362942,"id":"9130","kind":"VM","logical_id":"vm9130"},{"archive_size_bytes":3182,"id":"filebrowser","kind":"APP","logical_id":"app-filebrowser"},{"archive_size_bytes":156203,"id":"skladchik-reports-monitor","kind":"APP","logical_id":"app-skladchik-reports-monitor"},{"archive_size_bytes":270193546,"id":"immich","kind":"APP","logical_id":"app-immich"},{"archive_size_bytes":75522547,"id":"hbarhub","kind":"APP","logical_id":"app-hbarhub"}]} -{"clouds":[{"free_bytes":786104094818,"remote":"pve01-mail-01-crypt","total_bytes":1099511627776,"used_bytes":313407532958},{"free_bytes":786104030959,"remote":"pve01-mail-02-crypt","total_bytes":1099511627776,"used_bytes":313407596817}],"observed_at_epoch":1788564916,"schema":"homelab-backup-observability-snapshot-v1","targets":[{"archive_size_bytes":478357303,"id":"110","kind":"VM","logical_id":"vm110"},{"archive_size_bytes":369892675,"id":"111","kind":"VM","logical_id":"vm111"},{"archive_size_bytes":223672749,"id":"112","kind":"VM","logical_id":"vm112"},{"archive_size_bytes":221529138,"id":"113","kind":"VM","logical_id":"vm113"},{"archive_size_bytes":10614525791,"id":"130","kind":"VM","logical_id":"vm130"},{"archive_size_bytes":887338816,"id":"150","kind":"VM","logical_id":"vm150"},{"archive_size_bytes":3615630758,"id":"160","kind":"VM","logical_id":"vm160"},{"archive_size_bytes":6440615330,"id":"170","kind":"VM","logical_id":"vm170"},{"archive_size_bytes":2022982219,"id":"171","kind":"VM","logical_id":"vm171"},{"archive_size_bytes":1557482946,"id":"190","kind":"VM","logical_id":"vm190"},{"archive_size_bytes":10998362942,"id":"9130","kind":"VM","logical_id":"vm9130"},{"archive_size_bytes":3182,"id":"filebrowser","kind":"APP","logical_id":"app-filebrowser"},{"archive_size_bytes":156203,"id":"skladchik-reports-monitor","kind":"APP","logical_id":"app-skladchik-reports-monitor"},{"archive_size_bytes":270193546,"id":"immich","kind":"APP","logical_id":"app-immich"},{"archive_size_bytes":75522547,"id":"hbarhub","kind":"APP","logical_id":"app-hbarhub"},{"archive_size_bytes":8975409,"id":"control-bootstrap-v2","kind":"CONTROL","logical_id":"control-bootstrap-v2"}]} -{"clouds":[{"free_bytes":786104094818,"remote":"pve01-mail-01-crypt","total_bytes":1099511627776,"used_bytes":313407532958},{"free_bytes":786104030959,"remote":"pve01-mail-02-crypt","total_bytes":1099511627776,"used_bytes":313407596817}],"observed_at_epoch":1788568552,"schema":"homelab-backup-observability-snapshot-v1","targets":[{"archive_size_bytes":478357303,"id":"110","kind":"VM","logical_id":"vm110"},{"archive_size_bytes":369892675,"id":"111","kind":"VM","logical_id":"vm111"},{"archive_size_bytes":223672749,"id":"112","kind":"VM","logical_id":"vm112"},{"archive_size_bytes":221529138,"id":"113","kind":"VM","logical_id":"vm113"},{"archive_size_bytes":10614525791,"id":"130","kind":"VM","logical_id":"vm130"},{"archive_size_bytes":887338816,"id":"150","kind":"VM","logical_id":"vm150"},{"archive_size_bytes":3615630758,"id":"160","kind":"VM","logical_id":"vm160"},{"archive_size_bytes":6440615330,"id":"170","kind":"VM","logical_id":"vm170"},{"archive_size_bytes":2022982219,"id":"171","kind":"VM","logical_id":"vm171"},{"archive_size_bytes":1557482946,"id":"190","kind":"VM","logical_id":"vm190"},{"archive_size_bytes":10998362942,"id":"9130","kind":"VM","logical_id":"vm9130"},{"archive_size_bytes":3182,"id":"filebrowser","kind":"APP","logical_id":"app-filebrowser"},{"archive_size_bytes":156203,"id":"skladchik-reports-monitor","kind":"APP","logical_id":"app-skladchik-reports-monitor"},{"archive_size_bytes":270193546,"id":"immich","kind":"APP","logical_id":"app-immich"},{"archive_size_bytes":75522547,"id":"hbarhub","kind":"APP","logical_id":"app-hbarhub"},{"archive_size_bytes":8975409,"id":"control-bootstrap-v2","kind":"CONTROL","logical_id":"control-bootstrap-v2"}]} -{"clouds":[{"free_bytes":786104094818,"remote":"pve01-mail-01-crypt","total_bytes":1099511627776,"used_bytes":313407532958},{"free_bytes":786104030959,"remote":"pve01-mail-02-crypt","total_bytes":1099511627776,"used_bytes":313407596817}],"observed_at_epoch":1788572153,"schema":"homelab-backup-observability-snapshot-v1","targets":[{"archive_size_bytes":478357303,"id":"110","kind":"VM","logical_id":"vm110"},{"archive_size_bytes":369892675,"id":"111","kind":"VM","logical_id":"vm111"},{"archive_size_bytes":223672749,"id":"112","kind":"VM","logical_id":"vm112"},{"archive_size_bytes":221529138,"id":"113","kind":"VM","logical_id":"vm113"},{"archive_size_bytes":10614525791,"id":"130","kind":"VM","logical_id":"vm130"},{"archive_size_bytes":887338816,"id":"150","kind":"VM","logical_id":"vm150"},{"archive_size_bytes":3615630758,"id":"160","kind":"VM","logical_id":"vm160"},{"archive_size_bytes":6440615330,"id":"170","kind":"VM","logical_id":"vm170"},{"archive_size_bytes":2022982219,"id":"171","kind":"VM","logical_id":"vm171"},{"archive_size_bytes":1557482946,"id":"190","kind":"VM","logical_id":"vm190"},{"archive_size_bytes":10998362942,"id":"9130","kind":"VM","logical_id":"vm9130"},{"archive_size_bytes":3182,"id":"filebrowser","kind":"APP","logical_id":"app-filebrowser"},{"archive_size_bytes":156203,"id":"skladchik-reports-monitor","kind":"APP","logical_id":"app-skladchik-reports-monitor"},{"archive_size_bytes":270193546,"id":"immich","kind":"APP","logical_id":"app-immich"},{"archive_size_bytes":75522547,"id":"hbarhub","kind":"APP","logical_id":"app-hbarhub"},{"archive_size_bytes":8975409,"id":"control-bootstrap-v2","kind":"CONTROL","logical_id":"control-bootstrap-v2"}]} -{"clouds":[{"free_bytes":786104094818,"remote":"pve01-mail-01-crypt","total_bytes":1099511627776,"used_bytes":313407532958},{"free_bytes":786104030959,"remote":"pve01-mail-02-crypt","total_bytes":1099511627776,"used_bytes":313407596817}],"observed_at_epoch":1788575770,"schema":"homelab-backup-observability-snapshot-v1","targets":[{"archive_size_bytes":478357303,"id":"110","kind":"VM","logical_id":"vm110"},{"archive_size_bytes":369892675,"id":"111","kind":"VM","logical_id":"vm111"},{"archive_size_bytes":223672749,"id":"112","kind":"VM","logical_id":"vm112"},{"archive_size_bytes":221529138,"id":"113","kind":"VM","logical_id":"vm113"},{"archive_size_bytes":10614525791,"id":"130","kind":"VM","logical_id":"vm130"},{"archive_size_bytes":887338816,"id":"150","kind":"VM","logical_id":"vm150"},{"archive_size_bytes":3615630758,"id":"160","kind":"VM","logical_id":"vm160"},{"archive_size_bytes":6440615330,"id":"170","kind":"VM","logical_id":"vm170"},{"archive_size_bytes":2022982219,"id":"171","kind":"VM","logical_id":"vm171"},{"archive_size_bytes":1557482946,"id":"190","kind":"VM","logical_id":"vm190"},{"archive_size_bytes":10998362942,"id":"9130","kind":"VM","logical_id":"vm9130"},{"archive_size_bytes":3182,"id":"filebrowser","kind":"APP","logical_id":"app-filebrowser"},{"archive_size_bytes":156203,"id":"skladchik-reports-monitor","kind":"APP","logical_id":"app-skladchik-reports-monitor"},{"archive_size_bytes":270193546,"id":"immich","kind":"APP","logical_id":"app-immich"},{"archive_size_bytes":75522547,"id":"hbarhub","kind":"APP","logical_id":"app-hbarhub"},{"archive_size_bytes":8975409,"id":"control-bootstrap-v2","kind":"CONTROL","logical_id":"control-bootstrap-v2"}]} -{"clouds":[{"free_bytes":786104094818,"remote":"pve01-mail-01-crypt","total_bytes":1099511627776,"used_bytes":313407532958},{"free_bytes":786104030959,"remote":"pve01-mail-02-crypt","total_bytes":1099511627776,"used_bytes":313407596817}],"observed_at_epoch":1788579413,"schema":"homelab-backup-observability-snapshot-v1","targets":[{"archive_size_bytes":478357303,"id":"110","kind":"VM","logical_id":"vm110"},{"archive_size_bytes":369892675,"id":"111","kind":"VM","logical_id":"vm111"},{"archive_size_bytes":223672749,"id":"112","kind":"VM","logical_id":"vm112"},{"archive_size_bytes":221529138,"id":"113","kind":"VM","logical_id":"vm113"},{"archive_size_bytes":10614525791,"id":"130","kind":"VM","logical_id":"vm130"},{"archive_size_bytes":887338816,"id":"150","kind":"VM","logical_id":"vm150"},{"archive_size_bytes":3615630758,"id":"160","kind":"VM","logical_id":"vm160"},{"archive_size_bytes":6440615330,"id":"170","kind":"VM","logical_id":"vm170"},{"archive_size_bytes":2022982219,"id":"171","kind":"VM","logical_id":"vm171"},{"archive_size_bytes":1557482946,"id":"190","kind":"VM","logical_id":"vm190"},{"archive_size_bytes":10998362942,"id":"9130","kind":"VM","logical_id":"vm9130"},{"archive_size_bytes":3182,"id":"filebrowser","kind":"APP","logical_id":"app-filebrowser"},{"archive_size_bytes":156203,"id":"skladchik-reports-monitor","kind":"APP","logical_id":"app-skladchik-reports-monitor"},{"archive_size_bytes":270193546,"id":"immich","kind":"APP","logical_id":"app-immich"},{"archive_size_bytes":75522547,"id":"hbarhub","kind":"APP","logical_id":"app-hbarhub"},{"archive_size_bytes":8975409,"id":"control-bootstrap-v2","kind":"CONTROL","logical_id":"control-bootstrap-v2"}]} -{"clouds":[{"free_bytes":786104094818,"remote":"pve01-mail-01-crypt","total_bytes":1099511627776,"used_bytes":313407532958},{"free_bytes":786104030959,"remote":"pve01-mail-02-crypt","total_bytes":1099511627776,"used_bytes":313407596817}],"observed_at_epoch":1788583036,"schema":"homelab-backup-observability-snapshot-v1","targets":[{"archive_size_bytes":478357303,"id":"110","kind":"VM","logical_id":"vm110"},{"archive_size_bytes":369892675,"id":"111","kind":"VM","logical_id":"vm111"},{"archive_size_bytes":223672749,"id":"112","kind":"VM","logical_id":"vm112"},{"archive_size_bytes":221529138,"id":"113","kind":"VM","logical_id":"vm113"},{"archive_size_bytes":10614525791,"id":"130","kind":"VM","logical_id":"vm130"},{"archive_size_bytes":887338816,"id":"150","kind":"VM","logical_id":"vm150"},{"archive_size_bytes":3615630758,"id":"160","kind":"VM","logical_id":"vm160"},{"archive_size_bytes":6440615330,"id":"170","kind":"VM","logical_id":"vm170"},{"archive_size_bytes":2022982219,"id":"171","kind":"VM","logical_id":"vm171"},{"archive_size_bytes":1557482946,"id":"190","kind":"VM","logical_id":"vm190"},{"archive_size_bytes":10998362942,"id":"9130","kind":"VM","logical_id":"vm9130"},{"archive_size_bytes":3182,"id":"filebrowser","kind":"APP","logical_id":"app-filebrowser"},{"archive_size_bytes":156203,"id":"skladchik-reports-monitor","kind":"APP","logical_id":"app-skladchik-reports-monitor"},{"archive_size_bytes":270193546,"id":"immich","kind":"APP","logical_id":"app-immich"},{"archive_size_bytes":75522547,"id":"hbarhub","kind":"APP","logical_id":"app-hbarhub"},{"archive_size_bytes":8975409,"id":"control-bootstrap-v2","kind":"CONTROL","logical_id":"control-bootstrap-v2"}]} -{"clouds":[{"free_bytes":786104095252,"remote":"pve01-mail-01-crypt","total_bytes":1099511627776,"used_bytes":313407532524},{"free_bytes":786104031393,"remote":"pve01-mail-02-crypt","total_bytes":1099511627776,"used_bytes":313407596383}],"observed_at_epoch":1788588332,"schema":"homelab-backup-observability-snapshot-v1","targets":[{"archive_size_bytes":478357303,"id":"110","kind":"VM","logical_id":"vm110"},{"archive_size_bytes":369892675,"id":"111","kind":"VM","logical_id":"vm111"},{"archive_size_bytes":223672749,"id":"112","kind":"VM","logical_id":"vm112"},{"archive_size_bytes":221529138,"id":"113","kind":"VM","logical_id":"vm113"},{"archive_size_bytes":10614525791,"id":"130","kind":"VM","logical_id":"vm130"},{"archive_size_bytes":887338816,"id":"150","kind":"VM","logical_id":"vm150"},{"archive_size_bytes":3615630758,"id":"160","kind":"VM","logical_id":"vm160"},{"archive_size_bytes":6440615330,"id":"170","kind":"VM","logical_id":"vm170"},{"archive_size_bytes":2022982219,"id":"171","kind":"VM","logical_id":"vm171"},{"archive_size_bytes":1557482946,"id":"190","kind":"VM","logical_id":"vm190"},{"archive_size_bytes":10998362942,"id":"9130","kind":"VM","logical_id":"vm9130"},{"archive_size_bytes":3182,"id":"filebrowser","kind":"APP","logical_id":"app-filebrowser"},{"archive_size_bytes":156203,"id":"skladchik-reports-monitor","kind":"APP","logical_id":"app-skladchik-reports-monitor"},{"archive_size_bytes":270193546,"id":"immich","kind":"APP","logical_id":"app-immich"},{"archive_size_bytes":75522547,"id":"hbarhub","kind":"APP","logical_id":"app-hbarhub"},{"archive_size_bytes":8975409,"id":"control-bootstrap-v2","kind":"CONTROL","logical_id":"control-bootstrap-v2"}]} -{"clouds":[{"free_bytes":786104095252,"remote":"pve01-mail-01-crypt","total_bytes":1099511627776,"used_bytes":313407532524},{"free_bytes":786104031393,"remote":"pve01-mail-02-crypt","total_bytes":1099511627776,"used_bytes":313407596383}],"observed_at_epoch":1788591953,"schema":"homelab-backup-observability-snapshot-v1","targets":[{"archive_size_bytes":478357303,"id":"110","kind":"VM","logical_id":"vm110"},{"archive_size_bytes":369892675,"id":"111","kind":"VM","logical_id":"vm111"},{"archive_size_bytes":223672749,"id":"112","kind":"VM","logical_id":"vm112"},{"archive_size_bytes":221529138,"id":"113","kind":"VM","logical_id":"vm113"},{"archive_size_bytes":10614525791,"id":"130","kind":"VM","logical_id":"vm130"},{"archive_size_bytes":887338816,"id":"150","kind":"VM","logical_id":"vm150"},{"archive_size_bytes":3615630758,"id":"160","kind":"VM","logical_id":"vm160"},{"archive_size_bytes":6440615330,"id":"170","kind":"VM","logical_id":"vm170"},{"archive_size_bytes":2022982219,"id":"171","kind":"VM","logical_id":"vm171"},{"archive_size_bytes":1557482946,"id":"190","kind":"VM","logical_id":"vm190"},{"archive_size_bytes":10998362942,"id":"9130","kind":"VM","logical_id":"vm9130"},{"archive_size_bytes":3182,"id":"filebrowser","kind":"APP","logical_id":"app-filebrowser"},{"archive_size_bytes":156203,"id":"skladchik-reports-monitor","kind":"APP","logical_id":"app-skladchik-reports-monitor"},{"archive_size_bytes":270193546,"id":"immich","kind":"APP","logical_id":"app-immich"},{"archive_size_bytes":75522547,"id":"hbarhub","kind":"APP","logical_id":"app-hbarhub"},{"archive_size_bytes":8975409,"id":"control-bootstrap-v2","kind":"CONTROL","logical_id":"control-bootstrap-v2"}]} -{"clouds":[{"free_bytes":786104095252,"remote":"pve01-mail-01-crypt","total_bytes":1099511627776,"used_bytes":313407532524},{"free_bytes":786104031393,"remote":"pve01-mail-02-crypt","total_bytes":1099511627776,"used_bytes":313407596383}],"observed_at_epoch":1788595592,"schema":"homelab-backup-observability-snapshot-v1","targets":[{"archive_size_bytes":478357303,"id":"110","kind":"VM","logical_id":"vm110"},{"archive_size_bytes":369892675,"id":"111","kind":"VM","logical_id":"vm111"},{"archive_size_bytes":223672749,"id":"112","kind":"VM","logical_id":"vm112"},{"archive_size_bytes":221529138,"id":"113","kind":"VM","logical_id":"vm113"},{"archive_size_bytes":10614525791,"id":"130","kind":"VM","logical_id":"vm130"},{"archive_size_bytes":887338816,"id":"150","kind":"VM","logical_id":"vm150"},{"archive_size_bytes":3615630758,"id":"160","kind":"VM","logical_id":"vm160"},{"archive_size_bytes":6440615330,"id":"170","kind":"VM","logical_id":"vm170"},{"archive_size_bytes":2022982219,"id":"171","kind":"VM","logical_id":"vm171"},{"archive_size_bytes":1557482946,"id":"190","kind":"VM","logical_id":"vm190"},{"archive_size_bytes":10998362942,"id":"9130","kind":"VM","logical_id":"vm9130"},{"archive_size_bytes":3182,"id":"filebrowser","kind":"APP","logical_id":"app-filebrowser"},{"archive_size_bytes":156203,"id":"skladchik-reports-monitor","kind":"APP","logical_id":"app-skladchik-reports-monitor"},{"archive_size_bytes":270193546,"id":"immich","kind":"APP","logical_id":"app-immich"},{"archive_size_bytes":75522547,"id":"hbarhub","kind":"APP","logical_id":"app-hbarhub"},{"archive_size_bytes":8975409,"id":"control-bootstrap-v2","kind":"CONTROL","logical_id":"control-bootstrap-v2"}]} -{"clouds":[{"free_bytes":767079957393,"remote":"pve01-mail-01-crypt","total_bytes":1099511627776,"used_bytes":332431670383},{"free_bytes":768690899582,"remote":"pve01-mail-02-crypt","total_bytes":1099511627776,"used_bytes":330820728194}],"observed_at_epoch":1788600970,"schema":"homelab-backup-observability-snapshot-v1","targets":[{"archive_size_bytes":478357303,"id":"110","kind":"VM","logical_id":"vm110"},{"archive_size_bytes":369892675,"id":"111","kind":"VM","logical_id":"vm111"},{"archive_size_bytes":223672749,"id":"112","kind":"VM","logical_id":"vm112"},{"archive_size_bytes":221529138,"id":"113","kind":"VM","logical_id":"vm113"},{"archive_size_bytes":10591109796,"id":"130","kind":"VM","logical_id":"vm130"},{"archive_size_bytes":887338816,"id":"150","kind":"VM","logical_id":"vm150"},{"archive_size_bytes":3615630758,"id":"160","kind":"VM","logical_id":"vm160"},{"archive_size_bytes":6471807402,"id":"170","kind":"VM","logical_id":"vm170"},{"archive_size_bytes":1557482946,"id":"190","kind":"VM","logical_id":"vm190"},{"archive_size_bytes":10998362942,"id":"9130","kind":"VM","logical_id":"vm9130"},{"archive_size_bytes":3189,"id":"filebrowser","kind":"APP","logical_id":"app-filebrowser"},{"archive_size_bytes":156634,"id":"skladchik-reports-monitor","kind":"APP","logical_id":"app-skladchik-reports-monitor"},{"archive_size_bytes":270193548,"id":"immich","kind":"APP","logical_id":"app-immich"},{"archive_size_bytes":75598903,"id":"hbarhub","kind":"APP","logical_id":"app-hbarhub"},{"archive_size_bytes":8975409,"id":"control-bootstrap-v2","kind":"CONTROL","logical_id":"control-bootstrap-v2"}]} -{"clouds":[{"free_bytes":763045869314,"remote":"pve01-mail-01-crypt","total_bytes":1099511627776,"used_bytes":336465758462},{"free_bytes":763045805455,"remote":"pve01-mail-02-crypt","total_bytes":1099511627776,"used_bytes":336465822321}],"observed_at_epoch":1788604613,"schema":"homelab-backup-observability-snapshot-v1","targets":[{"archive_size_bytes":478357303,"id":"110","kind":"VM","logical_id":"vm110"},{"archive_size_bytes":369892675,"id":"111","kind":"VM","logical_id":"vm111"},{"archive_size_bytes":223672749,"id":"112","kind":"VM","logical_id":"vm112"},{"archive_size_bytes":221529138,"id":"113","kind":"VM","logical_id":"vm113"},{"archive_size_bytes":10591109796,"id":"130","kind":"VM","logical_id":"vm130"},{"archive_size_bytes":887338816,"id":"150","kind":"VM","logical_id":"vm150"},{"archive_size_bytes":6471807402,"id":"170","kind":"VM","logical_id":"vm170"},{"archive_size_bytes":2023943350,"id":"171","kind":"VM","logical_id":"vm171"},{"archive_size_bytes":1557482946,"id":"190","kind":"VM","logical_id":"vm190"},{"archive_size_bytes":10998362942,"id":"9130","kind":"VM","logical_id":"vm9130"},{"archive_size_bytes":3189,"id":"filebrowser","kind":"APP","logical_id":"app-filebrowser"},{"archive_size_bytes":156634,"id":"skladchik-reports-monitor","kind":"APP","logical_id":"app-skladchik-reports-monitor"},{"archive_size_bytes":270193548,"id":"immich","kind":"APP","logical_id":"app-immich"},{"archive_size_bytes":75598903,"id":"hbarhub","kind":"APP","logical_id":"app-hbarhub"},{"archive_size_bytes":8975409,"id":"control-bootstrap-v2","kind":"CONTROL","logical_id":"control-bootstrap-v2"}]} -{"clouds":[{"free_bytes":763036886053,"remote":"pve01-mail-01-crypt","total_bytes":1099511627776,"used_bytes":336474741723},{"free_bytes":763036822194,"remote":"pve01-mail-02-crypt","total_bytes":1099511627776,"used_bytes":336474805582}],"observed_at_epoch":1788609912,"schema":"homelab-backup-observability-snapshot-v1","targets":[{"archive_size_bytes":478357303,"id":"110","kind":"VM","logical_id":"vm110"},{"archive_size_bytes":369892675,"id":"111","kind":"VM","logical_id":"vm111"},{"archive_size_bytes":223672749,"id":"112","kind":"VM","logical_id":"vm112"},{"archive_size_bytes":221529138,"id":"113","kind":"VM","logical_id":"vm113"},{"archive_size_bytes":10591109796,"id":"130","kind":"VM","logical_id":"vm130"},{"archive_size_bytes":887338816,"id":"150","kind":"VM","logical_id":"vm150"},{"archive_size_bytes":3619770210,"id":"160","kind":"VM","logical_id":"vm160"},{"archive_size_bytes":6471807402,"id":"170","kind":"VM","logical_id":"vm170"},{"archive_size_bytes":2023943350,"id":"171","kind":"VM","logical_id":"vm171"},{"archive_size_bytes":1557482946,"id":"190","kind":"VM","logical_id":"vm190"},{"archive_size_bytes":10998362942,"id":"9130","kind":"VM","logical_id":"vm9130"},{"archive_size_bytes":3189,"id":"filebrowser","kind":"APP","logical_id":"app-filebrowser"},{"archive_size_bytes":156634,"id":"skladchik-reports-monitor","kind":"APP","logical_id":"app-skladchik-reports-monitor"},{"archive_size_bytes":270193548,"id":"immich","kind":"APP","logical_id":"app-immich"},{"archive_size_bytes":75598903,"id":"hbarhub","kind":"APP","logical_id":"app-hbarhub"},{"archive_size_bytes":8978589,"id":"control-bootstrap-v2","kind":"CONTROL","logical_id":"control-bootstrap-v2"}]} -{"clouds":[{"free_bytes":763036886053,"remote":"pve01-mail-01-crypt","total_bytes":1099511627776,"used_bytes":336474741723},{"free_bytes":763036822194,"remote":"pve01-mail-02-crypt","total_bytes":1099511627776,"used_bytes":336474805582}],"observed_at_epoch":1788613592,"schema":"homelab-backup-observability-snapshot-v1","targets":[{"archive_size_bytes":478357303,"id":"110","kind":"VM","logical_id":"vm110"},{"archive_size_bytes":369892675,"id":"111","kind":"VM","logical_id":"vm111"},{"archive_size_bytes":223672749,"id":"112","kind":"VM","logical_id":"vm112"},{"archive_size_bytes":221529138,"id":"113","kind":"VM","logical_id":"vm113"},{"archive_size_bytes":10591109796,"id":"130","kind":"VM","logical_id":"vm130"},{"archive_size_bytes":887338816,"id":"150","kind":"VM","logical_id":"vm150"},{"archive_size_bytes":3619770210,"id":"160","kind":"VM","logical_id":"vm160"},{"archive_size_bytes":6471807402,"id":"170","kind":"VM","logical_id":"vm170"},{"archive_size_bytes":2023943350,"id":"171","kind":"VM","logical_id":"vm171"},{"archive_size_bytes":1557482946,"id":"190","kind":"VM","logical_id":"vm190"},{"archive_size_bytes":10998362942,"id":"9130","kind":"VM","logical_id":"vm9130"},{"archive_size_bytes":3189,"id":"filebrowser","kind":"APP","logical_id":"app-filebrowser"},{"archive_size_bytes":156634,"id":"skladchik-reports-monitor","kind":"APP","logical_id":"app-skladchik-reports-monitor"},{"archive_size_bytes":270193548,"id":"immich","kind":"APP","logical_id":"app-immich"},{"archive_size_bytes":75598903,"id":"hbarhub","kind":"APP","logical_id":"app-hbarhub"},{"archive_size_bytes":8978589,"id":"control-bootstrap-v2","kind":"CONTROL","logical_id":"control-bootstrap-v2"}]} -{"clouds":[{"free_bytes":763036886053,"remote":"pve01-mail-01-crypt","total_bytes":1099511627776,"used_bytes":336474741723},{"free_bytes":763036822194,"remote":"pve01-mail-02-crypt","total_bytes":1099511627776,"used_bytes":336474805582}],"observed_at_epoch":1788618970,"schema":"homelab-backup-observability-snapshot-v1","targets":[{"archive_size_bytes":478357303,"id":"110","kind":"VM","logical_id":"vm110"},{"archive_size_bytes":369892675,"id":"111","kind":"VM","logical_id":"vm111"},{"archive_size_bytes":223672749,"id":"112","kind":"VM","logical_id":"vm112"},{"archive_size_bytes":221529138,"id":"113","kind":"VM","logical_id":"vm113"},{"archive_size_bytes":10591109796,"id":"130","kind":"VM","logical_id":"vm130"},{"archive_size_bytes":887338816,"id":"150","kind":"VM","logical_id":"vm150"},{"archive_size_bytes":3619770210,"id":"160","kind":"VM","logical_id":"vm160"},{"archive_size_bytes":6471807402,"id":"170","kind":"VM","logical_id":"vm170"},{"archive_size_bytes":2023943350,"id":"171","kind":"VM","logical_id":"vm171"},{"archive_size_bytes":1557482946,"id":"190","kind":"VM","logical_id":"vm190"},{"archive_size_bytes":10998362942,"id":"9130","kind":"VM","logical_id":"vm9130"},{"archive_size_bytes":3189,"id":"filebrowser","kind":"APP","logical_id":"app-filebrowser"},{"archive_size_bytes":156634,"id":"skladchik-reports-monitor","kind":"APP","logical_id":"app-skladchik-reports-monitor"},{"archive_size_bytes":270193548,"id":"immich","kind":"APP","logical_id":"app-immich"},{"archive_size_bytes":75598903,"id":"hbarhub","kind":"APP","logical_id":"app-hbarhub"},{"archive_size_bytes":8978589,"id":"control-bootstrap-v2","kind":"CONTROL","logical_id":"control-bootstrap-v2"}]} -{"clouds":[{"free_bytes":763036886053,"remote":"pve01-mail-01-crypt","total_bytes":1099511627776,"used_bytes":336474741723},{"free_bytes":763036822194,"remote":"pve01-mail-02-crypt","total_bytes":1099511627776,"used_bytes":336474805582}],"observed_at_epoch":1788624358,"schema":"homelab-backup-observability-snapshot-v1","targets":[{"archive_size_bytes":478357303,"id":"110","kind":"VM","logical_id":"vm110"},{"archive_size_bytes":369892675,"id":"111","kind":"VM","logical_id":"vm111"},{"archive_size_bytes":223672749,"id":"112","kind":"VM","logical_id":"vm112"},{"archive_size_bytes":221529138,"id":"113","kind":"VM","logical_id":"vm113"},{"archive_size_bytes":10591109796,"id":"130","kind":"VM","logical_id":"vm130"},{"archive_size_bytes":887338816,"id":"150","kind":"VM","logical_id":"vm150"},{"archive_size_bytes":3619770210,"id":"160","kind":"VM","logical_id":"vm160"},{"archive_size_bytes":6471807402,"id":"170","kind":"VM","logical_id":"vm170"},{"archive_size_bytes":2023943350,"id":"171","kind":"VM","logical_id":"vm171"},{"archive_size_bytes":1557482946,"id":"190","kind":"VM","logical_id":"vm190"},{"archive_size_bytes":10998362942,"id":"9130","kind":"VM","logical_id":"vm9130"},{"archive_size_bytes":3189,"id":"filebrowser","kind":"APP","logical_id":"app-filebrowser"},{"archive_size_bytes":156634,"id":"skladchik-reports-monitor","kind":"APP","logical_id":"app-skladchik-reports-monitor"},{"archive_size_bytes":270193548,"id":"immich","kind":"APP","logical_id":"app-immich"},{"archive_size_bytes":75598903,"id":"hbarhub","kind":"APP","logical_id":"app-hbarhub"},{"archive_size_bytes":8978589,"id":"control-bootstrap-v2","kind":"CONTROL","logical_id":"control-bootstrap-v2"}]} -{"clouds":[{"free_bytes":763036886053,"remote":"pve01-mail-01-crypt","total_bytes":1099511627776,"used_bytes":336474741723},{"free_bytes":763036822194,"remote":"pve01-mail-02-crypt","total_bytes":1099511627776,"used_bytes":336474805582}],"observed_at_epoch":1788628013,"schema":"homelab-backup-observability-snapshot-v1","targets":[{"archive_size_bytes":478357303,"id":"110","kind":"VM","logical_id":"vm110"},{"archive_size_bytes":369892675,"id":"111","kind":"VM","logical_id":"vm111"},{"archive_size_bytes":223672749,"id":"112","kind":"VM","logical_id":"vm112"},{"archive_size_bytes":221529138,"id":"113","kind":"VM","logical_id":"vm113"},{"archive_size_bytes":10591109796,"id":"130","kind":"VM","logical_id":"vm130"},{"archive_size_bytes":887338816,"id":"150","kind":"VM","logical_id":"vm150"},{"archive_size_bytes":3619770210,"id":"160","kind":"VM","logical_id":"vm160"},{"archive_size_bytes":6471807402,"id":"170","kind":"VM","logical_id":"vm170"},{"archive_size_bytes":2023943350,"id":"171","kind":"VM","logical_id":"vm171"},{"archive_size_bytes":1557482946,"id":"190","kind":"VM","logical_id":"vm190"},{"archive_size_bytes":10998362942,"id":"9130","kind":"VM","logical_id":"vm9130"},{"archive_size_bytes":3189,"id":"filebrowser","kind":"APP","logical_id":"app-filebrowser"},{"archive_size_bytes":156634,"id":"skladchik-reports-monitor","kind":"APP","logical_id":"app-skladchik-reports-monitor"},{"archive_size_bytes":270193548,"id":"immich","kind":"APP","logical_id":"app-immich"},{"archive_size_bytes":75598903,"id":"hbarhub","kind":"APP","logical_id":"app-hbarhub"},{"archive_size_bytes":8978589,"id":"control-bootstrap-v2","kind":"CONTROL","logical_id":"control-bootstrap-v2"}]} -{"clouds":[{"free_bytes":763036886053,"remote":"pve01-mail-01-crypt","total_bytes":1099511627776,"used_bytes":336474741723},{"free_bytes":763036822194,"remote":"pve01-mail-02-crypt","total_bytes":1099511627776,"used_bytes":336474805582}],"observed_at_epoch":1788633392,"schema":"homelab-backup-observability-snapshot-v1","targets":[{"archive_size_bytes":478357303,"id":"110","kind":"VM","logical_id":"vm110"},{"archive_size_bytes":369892675,"id":"111","kind":"VM","logical_id":"vm111"},{"archive_size_bytes":223672749,"id":"112","kind":"VM","logical_id":"vm112"},{"archive_size_bytes":221529138,"id":"113","kind":"VM","logical_id":"vm113"},{"archive_size_bytes":10591109796,"id":"130","kind":"VM","logical_id":"vm130"},{"archive_size_bytes":887338816,"id":"150","kind":"VM","logical_id":"vm150"},{"archive_size_bytes":3619770210,"id":"160","kind":"VM","logical_id":"vm160"},{"archive_size_bytes":6471807402,"id":"170","kind":"VM","logical_id":"vm170"},{"archive_size_bytes":2023943350,"id":"171","kind":"VM","logical_id":"vm171"},{"archive_size_bytes":1557482946,"id":"190","kind":"VM","logical_id":"vm190"},{"archive_size_bytes":10998362942,"id":"9130","kind":"VM","logical_id":"vm9130"},{"archive_size_bytes":3189,"id":"filebrowser","kind":"APP","logical_id":"app-filebrowser"},{"archive_size_bytes":156634,"id":"skladchik-reports-monitor","kind":"APP","logical_id":"app-skladchik-reports-monitor"},{"archive_size_bytes":270193548,"id":"immich","kind":"APP","logical_id":"app-immich"},{"archive_size_bytes":75598903,"id":"hbarhub","kind":"APP","logical_id":"app-hbarhub"},{"archive_size_bytes":8978589,"id":"control-bootstrap-v2","kind":"CONTROL","logical_id":"control-bootstrap-v2"}]} -{"clouds":[{"free_bytes":763036886053,"remote":"pve01-mail-01-crypt","total_bytes":1099511627776,"used_bytes":336474741723},{"free_bytes":763036822194,"remote":"pve01-mail-02-crypt","total_bytes":1099511627776,"used_bytes":336474805582}],"observed_at_epoch":1788638772,"schema":"homelab-backup-observability-snapshot-v1","targets":[{"archive_size_bytes":478357303,"id":"110","kind":"VM","logical_id":"vm110"},{"archive_size_bytes":369892675,"id":"111","kind":"VM","logical_id":"vm111"},{"archive_size_bytes":223672749,"id":"112","kind":"VM","logical_id":"vm112"},{"archive_size_bytes":221529138,"id":"113","kind":"VM","logical_id":"vm113"},{"archive_size_bytes":10591109796,"id":"130","kind":"VM","logical_id":"vm130"},{"archive_size_bytes":887338816,"id":"150","kind":"VM","logical_id":"vm150"},{"archive_size_bytes":3619770210,"id":"160","kind":"VM","logical_id":"vm160"},{"archive_size_bytes":6471807402,"id":"170","kind":"VM","logical_id":"vm170"},{"archive_size_bytes":2023943350,"id":"171","kind":"VM","logical_id":"vm171"},{"archive_size_bytes":1557482946,"id":"190","kind":"VM","logical_id":"vm190"},{"archive_size_bytes":10998362942,"id":"9130","kind":"VM","logical_id":"vm9130"},{"archive_size_bytes":3189,"id":"filebrowser","kind":"APP","logical_id":"app-filebrowser"},{"archive_size_bytes":156634,"id":"skladchik-reports-monitor","kind":"APP","logical_id":"app-skladchik-reports-monitor"},{"archive_size_bytes":270193548,"id":"immich","kind":"APP","logical_id":"app-immich"},{"archive_size_bytes":75598903,"id":"hbarhub","kind":"APP","logical_id":"app-hbarhub"},{"archive_size_bytes":8978589,"id":"control-bootstrap-v2","kind":"CONTROL","logical_id":"control-bootstrap-v2"}]} -{"clouds":[{"free_bytes":756548494504,"remote":"pve01-mail-01-crypt","total_bytes":1099511627776,"used_bytes":342963133272},{"free_bytes":763036822194,"remote":"pve01-mail-02-crypt","total_bytes":1099511627776,"used_bytes":336474805582}],"observed_at_epoch":1788644170,"schema":"homelab-backup-observability-snapshot-v1","targets":[{"archive_size_bytes":478357303,"id":"110","kind":"VM","logical_id":"vm110"},{"archive_size_bytes":369892675,"id":"111","kind":"VM","logical_id":"vm111"},{"archive_size_bytes":223672749,"id":"112","kind":"VM","logical_id":"vm112"},{"archive_size_bytes":221529138,"id":"113","kind":"VM","logical_id":"vm113"},{"archive_size_bytes":10591109796,"id":"130","kind":"VM","logical_id":"vm130"},{"archive_size_bytes":887338816,"id":"150","kind":"VM","logical_id":"vm150"},{"archive_size_bytes":3619770210,"id":"160","kind":"VM","logical_id":"vm160"},{"archive_size_bytes":2023943350,"id":"171","kind":"VM","logical_id":"vm171"},{"archive_size_bytes":1557482946,"id":"190","kind":"VM","logical_id":"vm190"},{"archive_size_bytes":10998362942,"id":"9130","kind":"VM","logical_id":"vm9130"},{"archive_size_bytes":3189,"id":"filebrowser","kind":"APP","logical_id":"app-filebrowser"},{"archive_size_bytes":156634,"id":"skladchik-reports-monitor","kind":"APP","logical_id":"app-skladchik-reports-monitor"},{"archive_size_bytes":270193548,"id":"immich","kind":"APP","logical_id":"app-immich"},{"archive_size_bytes":75598903,"id":"hbarhub","kind":"APP","logical_id":"app-hbarhub"},{"archive_size_bytes":8978589,"id":"control-bootstrap-v2","kind":"CONTROL","logical_id":"control-bootstrap-v2"}]} -{"clouds":[{"free_bytes":743574336966,"remote":"pve01-mail-01-crypt","total_bytes":1099511627776,"used_bytes":355937290810},{"free_bytes":743574273107,"remote":"pve01-mail-02-crypt","total_bytes":1099511627776,"used_bytes":355937354669}],"observed_at_epoch":1788649558,"schema":"homelab-backup-observability-snapshot-v1","targets":[{"archive_size_bytes":478357303,"id":"110","kind":"VM","logical_id":"vm110"},{"archive_size_bytes":369892675,"id":"111","kind":"VM","logical_id":"vm111"},{"archive_size_bytes":223672749,"id":"112","kind":"VM","logical_id":"vm112"},{"archive_size_bytes":221529138,"id":"113","kind":"VM","logical_id":"vm113"},{"archive_size_bytes":10588466039,"id":"130","kind":"VM","logical_id":"vm130"},{"archive_size_bytes":887338816,"id":"150","kind":"VM","logical_id":"vm150"},{"archive_size_bytes":3619770210,"id":"160","kind":"VM","logical_id":"vm160"},{"archive_size_bytes":6486807437,"id":"170","kind":"VM","logical_id":"vm170"},{"archive_size_bytes":2027528282,"id":"171","kind":"VM","logical_id":"vm171"},{"archive_size_bytes":1557482946,"id":"190","kind":"VM","logical_id":"vm190"},{"archive_size_bytes":10998362942,"id":"9130","kind":"VM","logical_id":"vm9130"},{"archive_size_bytes":3191,"id":"filebrowser","kind":"APP","logical_id":"app-filebrowser"},{"archive_size_bytes":157473,"id":"skladchik-reports-monitor","kind":"APP","logical_id":"app-skladchik-reports-monitor"},{"archive_size_bytes":270193543,"id":"immich","kind":"APP","logical_id":"app-immich"},{"archive_size_bytes":75669106,"id":"hbarhub","kind":"APP","logical_id":"app-hbarhub"},{"archive_size_bytes":8958389,"id":"control-bootstrap-v2","kind":"CONTROL","logical_id":"control-bootstrap-v2"}]} -{"clouds":[{"free_bytes":743574336966,"remote":"pve01-mail-01-crypt","total_bytes":1099511627776,"used_bytes":355937290810},{"free_bytes":743574273107,"remote":"pve01-mail-02-crypt","total_bytes":1099511627776,"used_bytes":355937354669}],"observed_at_epoch":1788653194,"schema":"homelab-backup-observability-snapshot-v1","targets":[{"archive_size_bytes":478357303,"id":"110","kind":"VM","logical_id":"vm110"},{"archive_size_bytes":369892675,"id":"111","kind":"VM","logical_id":"vm111"},{"archive_size_bytes":223672749,"id":"112","kind":"VM","logical_id":"vm112"},{"archive_size_bytes":221529138,"id":"113","kind":"VM","logical_id":"vm113"},{"archive_size_bytes":10588466039,"id":"130","kind":"VM","logical_id":"vm130"},{"archive_size_bytes":887338816,"id":"150","kind":"VM","logical_id":"vm150"},{"archive_size_bytes":3619770210,"id":"160","kind":"VM","logical_id":"vm160"},{"archive_size_bytes":6486807437,"id":"170","kind":"VM","logical_id":"vm170"},{"archive_size_bytes":2027528282,"id":"171","kind":"VM","logical_id":"vm171"},{"archive_size_bytes":1557482946,"id":"190","kind":"VM","logical_id":"vm190"},{"archive_size_bytes":10998362942,"id":"9130","kind":"VM","logical_id":"vm9130"},{"archive_size_bytes":3191,"id":"filebrowser","kind":"APP","logical_id":"app-filebrowser"},{"archive_size_bytes":157473,"id":"skladchik-reports-monitor","kind":"APP","logical_id":"app-skladchik-reports-monitor"},{"archive_size_bytes":270193543,"id":"immich","kind":"APP","logical_id":"app-immich"},{"archive_size_bytes":75669106,"id":"hbarhub","kind":"APP","logical_id":"app-hbarhub"},{"archive_size_bytes":8958389,"id":"control-bootstrap-v2","kind":"CONTROL","logical_id":"control-bootstrap-v2"}]} -{"clouds":[{"free_bytes":739949922534,"remote":"pve01-mail-01-crypt","total_bytes":1099511627776,"used_bytes":359561705242},{"free_bytes":739949858675,"remote":"pve01-mail-02-crypt","total_bytes":1099511627776,"used_bytes":359561769101}],"observed_at_epoch":1788656802,"schema":"homelab-backup-observability-snapshot-v1","targets":[{"archive_size_bytes":478357303,"id":"110","kind":"VM","logical_id":"vm110"},{"archive_size_bytes":369892675,"id":"111","kind":"VM","logical_id":"vm111"},{"archive_size_bytes":223672749,"id":"112","kind":"VM","logical_id":"vm112"},{"archive_size_bytes":221529138,"id":"113","kind":"VM","logical_id":"vm113"},{"archive_size_bytes":10588466039,"id":"130","kind":"VM","logical_id":"vm130"},{"archive_size_bytes":887338816,"id":"150","kind":"VM","logical_id":"vm150"},{"archive_size_bytes":3623527665,"id":"160","kind":"VM","logical_id":"vm160"},{"archive_size_bytes":6486807437,"id":"170","kind":"VM","logical_id":"vm170"},{"archive_size_bytes":2027528282,"id":"171","kind":"VM","logical_id":"vm171"},{"archive_size_bytes":1557482946,"id":"190","kind":"VM","logical_id":"vm190"},{"archive_size_bytes":10998362942,"id":"9130","kind":"VM","logical_id":"vm9130"},{"archive_size_bytes":3191,"id":"filebrowser","kind":"APP","logical_id":"app-filebrowser"},{"archive_size_bytes":157473,"id":"skladchik-reports-monitor","kind":"APP","logical_id":"app-skladchik-reports-monitor"},{"archive_size_bytes":270193543,"id":"immich","kind":"APP","logical_id":"app-immich"},{"archive_size_bytes":75669106,"id":"hbarhub","kind":"APP","logical_id":"app-hbarhub"},{"archive_size_bytes":8958389,"id":"control-bootstrap-v2","kind":"CONTROL","logical_id":"control-bootstrap-v2"}]} -HEALTH_FILE=homebox-backup.txt|mtime=2026-07-09 19:01:20.790193328 +0300|sha256=b388a7150af8169744f923e78e519408710c482afa7c2a3fcc334d1a1d3006e6 -STATUS=OK TS=2026-07-09T16:01:19Z TYPE=homebox-backup JOB=homelab-pve01-app-homebox-20260709T051713Z SOURCE=latest-appbackup SIZE=8873 LOCAL_BACKUP_DELETED=true JSON=/var/lib/homelab-backup/apps/latest-homebox.json NOTE=service-readiness-backup-proof -HEALTH_FILE=edge-vm-vzdump-backup.txt|mtime=2026-08-30 21:08:08.621601440 +0300|sha256=35e1b1b24eea8e1ca6506ef437b49606f1e691544f70965db8d30b9b075f9b12 -STATUS=FAIL -TS=2026-08-30T18:08:08Z -TYPE=backup-matrix-generated -JOB_ID=homelab_edge_vm_vzdump_backup -JOB_NAME=homelab-edge-vm-offhost-freshness -MATRIX_MODE=authoritative -TIMER=homelab-edge-vm-offhost-freshness.timer -SERVICE=homelab-edge-vm-offhost-freshness.service -TIMER_ENABLED=not-found -TIMER_ACTIVE=inactive -SOURCE_KIND=health_file -EVIDENCE_LEVEL=artifact_or_health -SERVICE_RESULT=success -SERVICE_EXEC_MAIN_STATUS=0 -SOURCE_HEALTH=/var/lib/homelab-health/edge-vm-offhost-freshness.txt -SOURCE_STATUS=FAIL -SOURCE_MTIME_UTC=2026-08-08T01:50:13Z -AGE_SECONDS=1959475 -MAX_AGE_SECONDS=172800 -FRESHNESS=STALE -LEGACY_SOURCE_PATH=/var/lib/homelab-health/edge-vm-vzdump-backup.txt -CANONICAL_JOB_ID=homelab_edge_vm_vzdump_backup -CANONICAL_STATE=/var/lib/homelab-backup-matrix/state.json -HEALTH_FILE=npmplus-kuma-config-backup.txt|mtime=2026-08-30 21:08:08.440410577 +0300|sha256=be5d4aaace4430857d61615081e8a8ac9fcc577aed6224356cf154114ae08250 -STATUS=FAIL -TS=2026-08-30T18:08:08Z -TYPE=backup-matrix-generated -JOB_ID=homelab_appbackup_npmplus_edge -JOB_NAME=homelab-appbackup-npmplus-edge -MATRIX_MODE=authoritative -TIMER=homelab-appbackup-npmplus-edge.timer -SERVICE=homelab-appbackup-npmplus-edge.service -TIMER_ENABLED=not-found -TIMER_ACTIVE=inactive -SOURCE_KIND=health_file -EVIDENCE_LEVEL=artifact_or_health -SERVICE_RESULT=success -SERVICE_EXEC_MAIN_STATUS=0 -SOURCE_HEALTH=/var/lib/homelab-backup-matrix/evidence/homelab_appbackup_npmplus_edge/0-latest-npmplus-edge.json -SOURCE_STATUS=OK -SOURCE_MTIME_UTC=2026-08-06T21:21:34Z -AGE_SECONDS=2061994 -MAX_AGE_SECONDS=172800 -FRESHNESS=STALE -LEGACY_SOURCE_PATH=/var/lib/homelab-health/npmplus-kuma-config-backup.txt -CANONICAL_JOB_ID=homelab_appbackup_npmplus_edge -CANONICAL_STATE=/var/lib/homelab-backup-matrix/state.json -HEALTH_FILE=backup-coverage-matrix.txt|mtime=2026-08-30 21:08:08.539509156 +0300|sha256=48b3b20a0dc225b93e0f7b8b1c2746649eb6f965d12f996030cc8ec8791604ff -STATUS=FAIL -TS=2026-08-30T18:08:08Z -TYPE=backup-matrix-generated -JOB_ID=homelab_backup_coverage_matrix -JOB_NAME=homelab-backup-coverage-matrix -MATRIX_MODE=authoritative -TIMER=homelab-backup-coverage-matrix.timer -SERVICE=homelab-backup-coverage-matrix.service -TIMER_ENABLED=not-found -TIMER_ACTIVE=inactive -SOURCE_KIND=health_file -EVIDENCE_LEVEL=artifact_or_health -SERVICE_RESULT=success -SERVICE_EXEC_MAIN_STATUS=0 -SOURCE_HEALTH=/var/lib/homelab-backup-matrix/evidence/homelab_backup_coverage_matrix/0-backup-coverage-matrix.txt -SOURCE_STATUS=OK -SOURCE_MTIME_UTC=2026-08-06T21:34:03Z -AGE_SECONDS=2061245 -MAX_AGE_SECONDS=172800 -FRESHNESS=STALE -LEGACY_SOURCE_PATH=/var/lib/homelab-health/backup-coverage-matrix.txt -CANONICAL_JOB_ID=homelab_backup_coverage_matrix -CANONICAL_STATE=/var/lib/homelab-backup-matrix/state.json -HEALTH_FILE=vm160-remote-backup.txt|mtime=2026-07-21 02:39:14.312087060 +0300|sha256=de5897d501d5bc25ca6fe964bf2643b2f8cac6957f9f577a1ae0cf4c68c3587e -STATUS=FAIL -SERVICE=homelab-vm160-remote-backup -TIMESTAMP_UTC=20260720T233913Z -VMID=160 -TARGET_DIR=/mnt/staging/vzdump/vm160-pve02-20260720T233913Z -RC=1 -HEALTH_FILE=vm-backup.txt|mtime=2026-07-24 07:43:04.241966168 +0300|sha256=78f0d0c132ea4b61be884501f342913fc13805900c083d82a59ddd30d067d980 -STATUS=OK TS=2026-07-24T04:43:04Z TYPE=vm-backup BAD=0 auto-vmct=OK edge-vm-backup=OK edge-vm-offhost=OK edge-vm-restore=OK auto-timer=active edge-freshness-timer=active -HEALTH_FILE=app-stacks-batch-backup.txt|mtime=2026-07-09 10:22:35.530260317 +0300|sha256=9218b8b34426e4ce79310c8713e165e9e1ebee79f426f29eb2efe155076b3c36 -STATUS=OK TS=2026-07-09T07:19:28Z TYPE=app-stacks-batch-backup SOURCE=backup-sla STATUS_SOURCE=OK NOTE=legacy-health-refresh-for-prometheus-readiness -HEALTH_FILE=blackbox-exporter-backup.txt|mtime=2026-07-09 11:04:19.961323675 +0300|sha256=2c46d63ab44ad8bd8f161a547103d8aef937643044b8b326e2b1ff609e9178d0 -STATUS=OK TS=2026-07-09T08:04:14Z TYPE=blackbox-exporter-backup JOB=homelab-pve01-app-blackbox-exporter-20260709T080414Z SOURCE=blackbox_exporter_config SIZE=512 SHA256=31d9f2a61d820629c0e173c0e1a24cd65e24d1ef55ba0670d7fda06225cb8146 LOCAL_BACKUP_DELETED=true NOTE=service-readiness-backup-proof -HEALTH_FILE=node-exporter-backup.txt|mtime=2026-07-09 11:14:05.435232618 +0300|sha256=9312b7986eeb48f06f3f3722fcf970624a423b4c398652d83226e59aaff4fefa -STATUS=OK TS=2026-07-09T08:13:40Z TYPE=node-exporter-backup JOB=homelab-pve01-app-node-exporter-20260709T081340Z SOURCE=multi_host_node_exporter_config_textfile SIZE=19018 SHA256=bb7db78e6e79c104f89c7735922e2a2e311fba242d34728e64cd60cefe69e647 LOCAL_BACKUP_DELETED=true PVE01_TEXTFILE_REPAIR=OK NOTE=service-readiness-backup-proof -HEALTH_FILE=stirling-pdf-backup.txt|mtime=2026-07-09 21:08:55.532973745 +0300|sha256=d292f237702fbead21f1597ac86dd610f8d3a6f70b81d18d2a3e3bbd8aced551 -STATUS=OK TS=2026-07-09T18:08:38Z TYPE=stirling-pdf-backup BAD=0 APP=stirling-pdf JOB=homelab-pve01-app-stirling-pdf-20260709T180838Z SIZE=40071 LOCAL_BACKUP_DELETED=true ENGINE=files JSON=/var/lib/homelab-backup/apps/latest-stirling-pdf.json -HEALTH_FILE=backup-audit.json|mtime=2026-09-23 23:47:36.448063169 +0300|sha256=b193b9fc04ed47f247d4c981468387ceff69cf1415e4295ec5b4de9e08d1b6fe -{"cloud":{"A":{"free":592249131673,"free_fraction":0.538647,"readable":true,"top_dirs":27,"total":1099511627776,"used":507262496103},"B":{"free":592249067814,"free_fraction":0.538647,"readable":true,"top_dirs":27,"total":1099511627776,"used":507262559962}},"cloud_b_restore_drill":true,"dr_proven":true,"generated_at_epoch":1790196452,"issues":[{"code":"RPO_BREACH","logical_id":"vm150","root_cause":"NO_FRESH_V2_OR_SEALED_QUORUM","severity":"HIGH"},{"code":"SOURCE_OFFHOST_MISSING","logical_id":"-","root_cause":"CR_BRANCH_SHA_NOT_EXACT","severity":"MEDIUM"}],"schema":"homelab-backup-audit-v3","source":{"local_head":"6401ea1e99988b9d2550de544c9e45bd7e829bdd","pushurl_disabled":true,"remote_head":null},"status":"RED","targets":[{"decision":"SKIP_FRESH","generation":"app-filebrowser-pve01-20260923T143005Z","kind":"APP","legacy_age_seconds":22640,"logical_id":"app-filebrowser","state":"FRESH"},{"decision":"SKIP_FRESH","generation":"app-hbarhub-pve01-20260923T143158Z","kind":"APP","legacy_age_seconds":22468,"logical_id":"app-hbarhub","state":"FRESH"},{"decision":"SKIP_FRESH","generation":"app-immich-pve01-20260923T143041Z","kind":"APP","legacy_age_seconds":22568,"logical_id":"app-immich","state":"FRESH"},{"decision":"SKIP_FRESH","generation":"app-skladchik-reports-monitor-pve01-20260923T143025Z","kind":"APP","legacy_age_seconds":22618,"logical_id":"app-skladchik-reports-monitor","state":"FRESH"},{"age_seconds":44470,"generation":"control-bootstrap-v2-pve01-20260916T120057Z","kind":"CONTROL","logical_id":"control-bootstrap-v2","state":"FRESH"},{"generation":"v2-ct110-20260923T125546Z-34408ffe4182","kind":"VM","legacy_age_seconds":1696083,"logical_id":"vm110","replica_count":2,"source":"V2","state":"FRESH","state_id":"ct110","v2_age_seconds":28097},{"generation":"v2-ct111-20260923T133222Z-0d3f5fae9c35","kind":"VM","legacy_age_seconds":null,"logical_id":"vm111","replica_count":2,"source":"V2","state":"FRESH","state_id":"ct111","v2_age_seconds":25955},{"generation":"v2-ct112-20260923T200600Z-c1b47cc67dad","kind":"VM","legacy_age_seconds":1734578,"logical_id":"vm112","replica_count":2,"source":"V2","state":"FRESH","state_id":"ct112","v2_age_seconds":2422},{"generation":"v2-ct113-20260923T203800Z-bd913a8e8930","kind":"VM","legacy_age_seconds":null,"logical_id":"vm113","replica_count":2,"source":"V2","state":"FRESH","state_id":"ct113","v2_age_seconds":497},{"generation":"v2-vm130-20260923T011558Z-c9498d37d3d7","kind":"VM","legacy_age_seconds":null,"logical_id":"vm130","replica_count":2,"source":"V2","state":"FRESH","state_id":"vm130","v2_age_seconds":69146},{"generation":"vm150-pve01-20260904T053517Z","kind":"VM","legacy_age_seconds":1696199,"logical_id":"vm150","replica_count":2,"source":"NONE","state":"STALE","state_id":"vm150","v2_age_seconds":74828},{"generation":"v2-vm160-20260923T014608Z-f76d7671af5a","kind":"VM","legacy_age_seconds":null,"logical_id":"vm160","replica_count":2,"source":"V2","state":"FRESH","state_id":"vm160","v2_age_seconds":68046},{"generation":"v2-vm170-20260923T035958Z-923adacbc94b","kind":"VM","legacy_age_seconds":640820,"logical_id":"vm170","replica_count":2,"source":"V2","state":"FRESH","state_id":"vm170","v2_age_seconds":59695},{"generation":"v2-vm171-20260923T021946Z-4d6e4f578075","kind":"VM","legacy_age_seconds":670352,"logical_id":"vm171","replica_count":2,"source":"V2","state":"FRESH","state_id":"vm171","v2_age_seconds":66249},{"generation":"v2-vm190-20260923T032759Z-314d9ba2ec35","kind":"VM","legacy_age_seconds":null,"logical_id":"vm190","replica_count":2,"source":"V2","state":"FRESH","state_id":"vm190","v2_age_seconds":62166},{"generation":"v2-vm9130-20260923T071932Z-c0492db6535b","kind":"VM","legacy_age_seconds":null,"logical_id":"vm9130","replica_count":2,"source":"V2","state":"FRESH","state_id":"vm9130","v2_age_seconds":47261}]} -HEALTH_FILE=crypto-portfolio-ui-preapply-backup.txt|mtime=2026-07-13 10:42:57.148769166 +0300|sha256=7b4b3faf518b57a01b5bb1302377fc1fb13ad9a206f9e8211edfd8bbbc0bf395 -STATUS=OK TS=2026-07-13T07:42:53Z TYPE=crypto-portfolio-ui-preapply-backup BAD=0 COMMAND_PHYSICAL_LINES=1 PREVIOUS_FAILURE=CORRECTED PREVIOUS_FAILURE_ROOT_CAUSE=INLINE_PYTHON_QUOTE_ERROR_IN_NONESSENTIAL_HELPER_PROOF BACKUP_SERVICE_RERUN=NO MANUAL_REHYDRATE_RERUN=NO CANONICAL_RESTORE_CHECK_RERUN=YES_READ_ONLY PRODUCTION_CHANGED=NO PRODUCTION_DATABASE_CHANGED=NO DESIRED_STATE_CHANGED=NO SERVICE_RESTARTED=NO CONTAINER_RECREATED=NO CLOUD_WRITE_RUN=NO_READ_ONLY_RESTORE_CHECK TARGET_HOST=core-apps.gram1.ru INPUT_INDEX_SHA256=8bbe88f5f420fe429d479c598e1f3f904521eb166d4707e32e579851e84c205e CANDIDATE_INDEX_SHA256=e497a36d8549654c6d451360a11a2ad714e19c23e6d384330a5436f7d32267cb BACKUP_JOB=homelab-pve01-app-crypto-portfolio-20260713T071127Z BACKUP_STATUS=success BACKUP_SCHEMA=homelab-app-backup-manifest-v2-flat BACKUP_LAYOUT=flat-v1 BACKUP_PATH_REL=hb/a/cp-260713-071127 BACKUP_DATABASE_INTEGRITY=ok BACKUP_FILE_COUNT=19 BACKUP_ARCHIVE_SIZE_BYTES=136632 BACKUP_ARCHIVE_SHA256=b29eb55820ec46c474f51e995e92e371251b6df717ac98182d807d1dbf8473f5 BACKUP_LOCAL_STAGING_DELETED=YES CANONICAL_TWO_CLOUD_RESTORE_CHECK=OK APPBACKUPCTL_RESTORE_CHECK=OK_BOTH_CLOUDS ROLLBACK_PATH=/var/lib/homelab-incidents/crypto-ui-center-alert-currency-prechange-20260713T071125Z ROLLBACK_MANIFEST_SHA256=0ef56c8f021f549777131ffe878cd5a263bdfb6f1dfb8bb080cefdd554573aca ROLLBACK_INDEX_SHA256=8bbe88f5f420fe429d479c598e1f3f904521eb166d4707e32e579851e84c205e PRODUCTION_UNCHANGED=YES PRODUCTION_HEALTH=200 VM170_FAILED_UNITS=0 PVE01_UNRELATED_FAILED_BASELINE_UNCHANGED=YES CHART_HEALTH_TIMER=ACTIVE BACKUP_TIMER=ACTIVE DESIRED_STATE_HEAD=80763a6c62b94cde4f3293f3b529980f76919d25 DESIRED_STATE_CLEAN=YES DESIRED_STATE_REMOTE_COUNT=0 READY_FOR_UI_PRODUCTION_APPLY=YES PREAPPLY_ROOT=/var/lib/homelab-change-sets/crypto-portfolio-ui-preapply-backup-canonical-close/20260713T074253Z -HEALTH_FILE=backup-matrix-vm-health-feedback-fix.txt|mtime=2026-07-13 20:29:02.896780731 +0300|sha256=b359a93e09405221e25670600e4b45503ac30c7a8ada2f4e06f0cd9734a6e803 -STATUS=OK TS=2026-07-13T17:29:02Z TYPE=backup-matrix-vm-health-feedback-fix BAD=0 VM_BACKUP_HEALTH_TIMER=homelab-edge-vm-offhost-freshness.timer OLD_TIMER_REENABLED=NO MATRIX_STATUS=OK JOBS_TOTAL=35 JOBS_BAD=0 JOBS_UNMAPPED=0 JOBS_DISABLED=4 READ_WRITE_OVERLAP=0 HEAVY_BACKUP_RUN=NO LIVE_SCRIPT_SHA256=a795ad5cacd48ac72e85c3a7908ad456d1e7c563ef4399e6957725146463faf2 MATRIX_CONFIG_SHA256=7613bcb25d32e61fac114da13af81e38354ec22a012a6deeaa168ee4cc454f87 MATRIX_STATE_SHA256=75f8c749e25df7842ea3be2679119b922c94c2f5ea99700b56c823cfbaedd539 ROLLBACK=/var/lib/homelab-incidents/vm-backup-health-matrix-feedback-fix-20260713T172901Z -HEALTH_FILE=beszel-two-cloud-backup.txt|mtime=2026-07-10 22:05:25.857040628 +0300|sha256=31278a684937710c66cdabb2c4eff0c0fda078e7fef976ab8f4c49ea5778a5ca -STATUS=OK TS=2026-07-10T19:05:25Z TYPE=beszel-two-cloud-backup BAD=0 JOB=homelab-pve01-app-beszel-20260710T190453Z ARCHIVE_SIZE_BYTES=20068 LOCAL_STAGING_DELETED=true OFFHOST=OK RESTORE=OK -HEALTH_FILE=cluster-admin-incident-engine-backup.txt|mtime=2026-07-11 02:30:49.625263708 +0300|sha256=5f56ac6b48e94a81f7a9ffa073525dc255b1fede76fcc04f7b629e0f51cb98ed -STATUS=OK TS=2026-07-10T23:30:49Z TYPE=cluster-admin-incident-engine-backup BAD=0 JOB=homelab-pve01-app-cluster-admin-incident-engine-20260710T232958Z ARCHIVE_SIZE_BYTES=157380 LOCAL_STAGING_DELETED=true OFFHOST=OK RESTORE=OK POST_FIX_CODE=included -HEALTH_FILE=backup-unified-audit.json|mtime=2026-09-21 07:36:42.412067103 +0300|sha256=5da55c119b77511b150a6c2c7380842551293e5d3d12e807ab25291a4d9031b6 -{ - "schema": "homelab-backup-unified-audit-v1", - "task_id": "backup-unified-monitoring-v1", - "change_id": "CR-2026-0032", - "updated_at_utc": "2026-09-21T04:36:42Z", - "audit_status": "OK", - "backup_status": "RED", - "risk_severity": "CRITICAL", - "risk_reasons": [ - "SOURCE_DRIFT", - "INFRASTRUCTURE_INVARIANT", - "RPO_BREACH", - "CAPACITY_CRITICAL", - "RESUME_PENDING" - ], - "dr_readiness": "PROVEN", - "dr_proof_status": "PROVEN", - "control_freshness": "STALE_PROVEN", - "lifecycle_status": "CLOSED_PROVEN", - "vm_count": 11, - "app_count": 4, - "fresh_count": 4, - "stale_count": 10, - "resume_count": 1, - "unverified_count": 0, - "cloud_ok_count": 2, - "scheduler_ok_count": 6, - "source_ok_count": 28, - "source_drift_count": 1, - "payload_leaks": 0, - "control_status": "STALE_PROVEN", - "control_source_coverage": "RESTORE_EVIDENCE", - "staging_status": "OK", - "staging_bad_count": 0, - "staging_warn_count": 0, - "staging_node_count": 3, - "escrow_status": "OK", - "retention_status": "ACTIVE_OWNER_AWARE", - "historical_gc_proven": false, - "owner_aware_gc_proven": true, - "observability_status": "ATTENTION", - "size_anomaly_count": 0, - "size_learning_count": 0, - "capacity_status": "CRIT", - "capacity_worst_used_percent": 74, - "capacity_worst_days_to_80": 3, - "observability_history_snapshot_count": 336, - "provider_worm_proven": false, - "provider_worm_skipped_by_policy": true, - "clean_machine_bootstrap_proven": true, -HEALTH_FILE=legacy-monitoring-backups-migrated.txt|mtime=2026-07-10 23:00:03.116834953 +0300|sha256=3d8a60e0d8aeb25aed50f6eb2a92df4ef2f544b7409711f0afec378099b635c7 -STATUS=OK TS=2026-07-10T20:00:03Z TYPE=legacy-monitoring-backups-migrated BAD=0 APPS=6 RESTORE_FAILURES=0 ACTIVE_LEGACY_STATES=0 OFFHOST=OK RESTORE=OK -HEALTH_FILE=backup-boot-drill.json|mtime=2026-09-20 07:40:22.416980264 +0300|sha256=5cf6ed79fdec1f2901a868b05a57f17d1388189dcc2fb32e191b0ee4fd1f5d97 -{ - "drill_executed": false, - "reason": "BACKUP_LOCK_BUSY", - "schema": "homelab-backup-boot-drill-health-v1", - "status": "DEFERRED", - "updated_at_utc": "2026-09-20T04:40:22Z" -} -HEALTH_FILE=vm170-vm171-full-backup.txt|mtime=2026-08-30 21:08:08.908433073 +0300|sha256=dec2afb6c8b47c0c6be4f5a98e226ac3285524bc997cd93dd73942314e4367ee -STATUS=FAIL -TS=2026-08-30T18:08:08Z -TYPE=backup-matrix-generated -JOB_ID=homelab_vm_full_backup -JOB_NAME=homelab-vm-full-backup -MATRIX_MODE=authoritative -TIMER=homelab-vm-full-backup.timer -SERVICE=homelab-vm-full-backup.service -TIMER_ENABLED=not-found -TIMER_ACTIVE=inactive -SOURCE_KIND=health_file -EVIDENCE_LEVEL=artifact_or_health -SERVICE_RESULT=success -SERVICE_EXEC_MAIN_STATUS=0 -SOURCE_HEALTH=/var/lib/homelab-backup-matrix/evidence/homelab_vm_full_backup/0-vm170-vm171-full-backup.txt -SOURCE_STATUS=OK -SOURCE_MTIME_UTC=2026-07-24T00:10:18Z -AGE_SECONDS=3261470 -MAX_AGE_SECONDS=172800 -FRESHNESS=STALE -LEGACY_SOURCE_PATH=/var/lib/homelab-health/vm170-vm171-full-backup.txt -CANONICAL_JOB_ID=homelab_vm_full_backup -CANONICAL_STATE=/var/lib/homelab-backup-matrix/state.json -HEALTH_FILE=uptime-kuma-backup.txt|mtime=2026-07-09 23:39:50.594635691 +0300|sha256=62270c278bd63d2068eca9c0ee8a8036436328334a9393b357353eb8d52cc079 -STATUS=OK TS=2026-07-09T20:39:29Z TYPE=uptime-kuma-backup BAD=0 APP=uptime-kuma JOB=homelab-pve01-app-uptime-kuma-20260709T072138Z SIZE=99851064 LOCAL_BACKUP_DELETED=true JSON=/var/lib/homelab-backup/apps/latest-uptime-kuma.json -HEALTH_FILE=calibre-web-backup.txt|mtime=2026-07-09 21:46:20.933829215 +0300|sha256=2e841d7edd46668242924dae527480ead9c298abeb30a4c96734c382b09c90d7 -STATUS=OK TS=2026-07-09T18:46:20Z TYPE=calibre-web-backup BAD=0 APP=calibre-web JOB=homelab-pve01-app-calibre-web-20260709T051713Z SIZE=6630 LOCAL_BACKUP_DELETED=true JSON=/var/lib/homelab-backup/apps/latest-calibre-web.json -HEALTH_FILE=syncthing-backup.txt|mtime=2026-07-09 22:55:13.502354244 +0300|sha256=f9afb07a38c097b42a37b5a710917fafa6d31c943953d8daf6db72b7b91d9c24 -STATUS=OK TS=2026-07-09T19:55:12Z TYPE=syncthing-backup BAD=0 APP=syncthing JOB=homelab-pve01-app-syncthing-20260709T051713Z SIZE=6732 LOCAL_BACKUP_DELETED=true JSON=/var/lib/homelab-backup/apps/latest-syncthing.json -HEALTH_FILE=dozzle-backup.txt|mtime=2026-07-09 11:32:11.068758347 +0300|sha256=cc3f0d96c68c610cad9c1c38c70c89387617d43e1df6e0d304e4b7ae6b0464bb -STATUS=OK TS=2026-07-09T08:32:09Z TYPE=dozzle-backup JOB=homelab-pve01-app-dozzle-20260709T083209Z SOURCE=dozzle_stack_config SIZE=1328 SHA256=a571ee6f6012b25c16cdf15693dba29d8dd3acc3ce872b52f5c34edb81aa1ec8 LOCAL_BACKUP_DELETED=true NOTE=service-readiness-backup-proof -HEALTH_FILE=vm-backup-policy.txt|mtime=2026-08-30 21:08:08.889182826 +0300|sha256=947eef16fbfd3815d6f823c3bc990415a33c5611342bc0d32fe959cb9e95b799 -STATUS=FAIL -TS=2026-08-30T18:08:08Z -TYPE=backup-matrix-generated -JOB_ID=homelab_vm_backup_policy -JOB_NAME=homelab-vm-backup-policy -MATRIX_MODE=authoritative -TIMER=homelab-vm-backup-policy.timer -SERVICE=homelab-vm-backup-policy.service -TIMER_ENABLED=not-found -TIMER_ACTIVE=inactive -SOURCE_KIND=health_file -EVIDENCE_LEVEL=artifact_or_health -SERVICE_RESULT=success -SERVICE_EXEC_MAIN_STATUS=0 -SOURCE_HEALTH=/var/lib/homelab-backup-matrix/evidence/homelab_vm_backup_policy/0-vm-backup-policy.txt -SOURCE_STATUS=OK -SOURCE_MTIME_UTC=2026-07-24T05:26:24Z -AGE_SECONDS=3242504 -MAX_AGE_SECONDS=172800 -FRESHNESS=STALE -LEGACY_SOURCE_PATH=/var/lib/homelab-health/vm-backup-policy.txt -CANONICAL_JOB_ID=homelab_vm_backup_policy -CANONICAL_STATE=/var/lib/homelab-backup-matrix/state.json -HEALTH_FILE=netbird-peers.txt|mtime=2026-09-23 23:58:22.533880166 +0300|sha256=77fdd7924a7623499cc18560449d2190a7123fe7f0277c469a8a1360f440c869 -STATUS=ERROR -SERVICE=netbird-peers -TIMESTAMP_UTC=20260923T205759Z -ATTEMPT_USED=2 -pve01 service=active/enabled wt0_ip=[IP]/16 management=no signal=no peers=2/11 Connected -pve02 service=active/enabled wt0_ip=[IP]/16 management=no signal=no peers=2/11 Connected -pve03 service=active/enabled wt0_ip=[IP]/16 management=no signal=no peers=2/11 Connected -edge-vm service=active/enabled wt0_ip=[IP]/16 management=yes signal=yes peers=2/11 Connected -HEALTH_FILE=hapusya-site-backup-path-mapping.txt|mtime=2026-07-13 15:10:58.138221543 +0300|sha256=fb9db1095347b4db4dc7e37d1085cb758a5608ff8c6f3b934da00f44a905142c -STATUS=OK TS=2026-07-13T12:10:58Z TYPE=hapusya-site-backup-path-mapping BAD=0 MAP_SHA256=815b52dcedf7e3e92e07939bade32c7e8ed6acc0f9e064b65db6daec9f559c63 SHORT=hps LEGACY_STATE_PATH=pve01/apps/hapusya-site/homelab-pve01-app-hapusya-site-20260713T115259Z DESIRED_STATE_SNAPSHOT=/srv/homelab-desired-state/config/homelab-backup-paths.map RESTORE_CHECK=OK_BOTH_CLOUDS RETENTION=OK PRODUCTION_CHANGED=YES_COMMIT_POINT_PRESERVED -HEALTH_FILE=crowdsec-capi-netbird-final.txt|mtime=2026-07-02 22:30:48.260143650 +0300|sha256=20b03407e972e39c3a8b7e8ca008bf4a4b6928ed3e0e9a822be886c3cd63fae0 -STATUS=OK -TS=20260702T193048Z -CHECK=crowdsec-capi-netbird-final -DETAIL=CrowdSec CAPI/LAPI stable through NetBird exit route relay [IP]; WARP absent; route smoke OK; documentation and error register updated. -EDGE=edge-vm -EDGE_IP=[IP] -EDGE_NETBIRD_IP=[IP] -PRIMARY_EXIT=relay.netbird.selfhosted -PRIMARY_EXIT_NETBIRD_IP=[IP] -BACKUP_EXIT=mail.netbird.selfhosted -BACKUP_EXIT_NETBIRD_IP=[IP] -TRACE_IP=[IP] -TRACE_COLO=OTP -PROOFS=771AC_REGISTER_CROWDSEC_CAPI_AFTER_EXIT_ROUTE,771AD_FINAL_CROWDSEC_CAPI_NETBIRD_CLOSURE,771AE_CROWDSEC_CAPI_SOAK_FINAL,771AF_CROWDSEC_CAPI_PROOF_INDEX -HEALTH_FILE=cluster-admin-stage3-preapply-backup.txt|mtime=2026-07-13 06:15:56.239623410 +0300|sha256=95632c844cbdafe9be502b9164b1ec4351e55a1a0f1ae55668426c8affaa657c -STATUS=OK TS=2026-07-13T03:15:54Z TYPE=cluster-admin-stage3-preapply-backup BAD=0 COMMAND_PHYSICAL_LINES=1 PRODUCTION_CHANGED=EVIDENCE_ONLY LIVE_APPLICATION_CODE_CHANGED=NO DESIRED_STATE_CHANGED=NO SERVICE_RESTARTED=NO CLOUD_WRITE_RUN=NO_REUSE_VERIFIED_EXISTING_BACKUP APP_PRECHANGE_SHA256=c2748b8f96df842f491b5d2790790594222549853d0e2336cf60ecd6d4de0020 APP_CANDIDATE_SHA256=672d6a2975dd0e093788e7eb81a76e7f901ea6374ca4092a25ba0091aba1cb50 ROLLBACK_CHECKPOINT=VERIFIED REMOTE_CHECKPOINT_DIR=/var/lib/cluster-admin-stage3-preapply/20260713T030540Z CHECKPOINT_MANIFEST_SHA256=ede8c349a3456384f11460477d7cabc1a2aa0d18189ef3aa772471f2ebb26c77 CHECKPOINT_VALIDATION_WORKDIR=CORRECTED APPBACKUP=VERIFIED APPBACKUP_JOB=homelab-pve01-app-cluster-admin-incident-engine-20260713T031116Z APPBACKUP_SIZE_BYTES=1767199 APPBACKUP_RUN_LOG_EMPTY=ACCEPTED APPBACKUP_RESTORE_CHECK=OK APPBACKUP_ROUTING=VERIFIED LIVE_APP_UNCHANGED=YES PRODUCTION_RUNTIME_HEALTH=OK DESIRED_STATE_HEAD=8f888e5f2063e49da918b31584c9ea4bcf53af3a PREAPPLY_ROOT=/var/lib/homelab-stage3-preapply-appbackup/20260713T031116Z READY_FOR_STAGE3_PRODUCTION_APPLY=YES -HEALTH_FILE=node-red-backup.txt|mtime=2026-07-09 22:46:45.732436363 +0300|sha256=cf53c240e74a9f8670ab0af52681ce9de4e45f332e70fc142925c357eaf9f00c -STATUS=OK TS=2026-07-09T19:46:42Z TYPE=node-red-backup BAD=0 APP=node-red JOB=homelab-pve01-app-node-red-20260709T051713Z SIZE=10521 LOCAL_BACKUP_DELETED=true JSON=/var/lib/homelab-backup/apps/latest-node-red.json -HEALTH_FILE=proxmox-pve01-backup.txt|mtime=2026-07-09 15:29:41.985138298 +0300|sha256=2b8ca75eeec121d7e2b0c6a309d43798da5f864b57fef327f37a56b9ced3b5a8 -STATUS=OK TS=2026-07-09T12:29:38Z TYPE=proxmox-pve01-backup JOB=homelab-pve01-app-proxmox-pve01-20260709T122938Z SOURCE=safe_pve01_config_facts SIZE=4314 LOCAL_BACKUP_DELETED=true JSON=/var/lib/homelab-backup/apps/latest-proxmox-pve01.json NOTE=service-readiness-backup-proof -HEALTH_FILE=service-readiness-backup-health.txt|mtime=2026-07-09 09:41:40.097970342 +0300|sha256=f8074ea30fd108c0db7c85caebaaaa4a3600ea49fbea717f372bf5e6218753b4 -STATUS=OK TS=2026-07-09T06:41:40Z TYPE=service-readiness-final BAD=0 SERVICE=backup-health HOMEPAGE=OK NPMPLUS=OK KUMA=OK ENDPOINTS=LOCAL_LAN_PUBLIC_OK CORE_GATES=OK LISTENERS=OK URL=https://backup.gram1.ru/backup.txt OVERALL=https://backup.gram1.ru/overall.txt -HEALTH_FILE=hapusya-site-backup.txt|mtime=2026-07-13 15:10:58.137221528 +0300|sha256=5f8d4f49cf5cf6dba74912589dcd3c7fa0542171bfa31c649f6d94292fb8bb64 -STATUS=OK TS=2026-07-13T12:10:58Z TYPE=hapusya-site-backup JOB=homelab-pve01-app-hapusya-site-20260713T115259Z ARCHIVE_SHA256=bf284292974600384c43c47fc99be6fedf0ebd68f722b402a7260f1df680d167 ARCHIVE_SIZE_BYTES=7895834 LOCAL_BACKUP_DELETED=true TWO_CLOUDS=OK ROUTE_SHA256=02d65cb0082a04cae282126cc829a002f171e863291c18ba04d788dc2d6282ca UNIT_SHA256=b3b07137b25ac0a0b4bdadadad654cf209387ade86a2ef2cdca58e8a7c0cc212 RELATIVE_TREE_SHA256=9dadf21da041bc4d6ee4a42f5b3ac7719c16bcc39052de32af895c6cd28286cb -HEALTH_FILE=backup.txt|mtime=2026-07-24 07:04:41.721444174 +0300|sha256=88d791f9955261ebe5a65303dc418c70dca25ba23c29a92769a6785a9cd9e5bb -STATUS=OK -REASON=latest backup exists, fresh, complete, and sha256 verified -LATEST=/mnt/staging/vzdump/auto-full-backup-20260719T004537Z -AGE_HOURS=123 -START=2026-07-24T07:04:28+03:00 -END=2026-07-24T07:04:41+03:00 -RC=0 -HEALTH_FILE=npmplus-edge-backup.txt|mtime=2026-07-09 13:02:56.571425663 +0300|sha256=83e492c51913e884cad62305314e153c49bad522b9d65cddaefe337b7ed10e2d -STATUS=OK TS=2026-07-09T10:02:52Z TYPE=npmplus-edge-backup JOB=homelab-pve01-app-npmplus-edge-20260709T071928Z SOURCE=latest-appbackup SIZE=50316 LOCAL_BACKUP_DELETED=true JSON=/var/lib/homelab-backup/apps/latest-npmplus-edge.json NOTE=service-readiness-backup-proof -HEALTH_FILE=p1-safe-data-backup.txt|mtime=2026-07-07 01:09:37.332826625 +0300|sha256=c1f7e49cbb188b0c1b13aed8508d267d47212490e92b6cb85a3b70ab167fd13d -HEALTH_FILE=gitea-backup.txt|mtime=2026-08-30 21:08:08.480366809 +0300|sha256=5474fdacc557aef613bf14e7830fac12d9e616320519074a228a0a81ff6e8e68 -STATUS=FAIL -TS=2026-08-30T18:08:08Z -TYPE=backup-matrix-generated -JOB_ID=homelab_appbackup_sqlite_batch -JOB_NAME=homelab-appbackup-sqlite-batch -MATRIX_MODE=authoritative -TIMER=homelab-appbackup-sqlite-batch.timer -SERVICE=homelab-appbackup-sqlite-batch.service -TIMER_ENABLED=not-found -TIMER_ACTIVE=inactive -SOURCE_KIND=systemd -EVIDENCE_LEVEL=execution_only -SERVICE_RESULT=success -SERVICE_EXEC_MAIN_STATUS=0 -SOURCE_HEALTH=systemd:homelab-appbackup-sqlite-batch.service -SOURCE_STATUS=OK -SOURCE_MTIME_UTC=2026-07-24T01:46:28Z -AGE_SECONDS=3255700 -MAX_AGE_SECONDS=172800 -FRESHNESS=STALE -LEGACY_SOURCE_PATH=/var/lib/homelab-health/gitea-backup.txt -CANONICAL_JOB_ID=homelab_appbackup_sqlite_batch -CANONICAL_STATE=/var/lib/homelab-backup-matrix/state.json -HEALTH_FILE=homeassistant-backup.txt|mtime=2026-07-09 22:04:04.073386425 +0300|sha256=03ce1462f536f5c3e3eb7b5224465292d1810b063946f859eb6e504eb5bd0a19 -STATUS=OK TS=2026-07-09T19:04:00Z TYPE=homeassistant-backup BAD=0 APP=homeassistant JOB=homelab-pve01-app-homeassistant-20260709T051713Z SIZE=1855722 LOCAL_BACKUP_DELETED=true JSON=/var/lib/homelab-backup/apps/latest-homeassistant.json -HEALTH_FILE=backup-matrix-desired-state-persistence.txt|mtime=2026-07-13 20:40:37.065354772 +0300|sha256=3d5bdf5d8ec15131c0d801287f324cbe0b6099e1fd079776ce0e3df8a5fa6144 -STATUS=OK TS=2026-07-13T17:40:37Z TYPE=backup-matrix-desired-state-persistence BAD=0 GENERATED_HEAD=af26ec3c4829b1330d49656205ac580ce1706263 GENERATED_PARENT=25568fab24c0594418b3676dc511c8234b4e1ae5 GENERATOR_SHA256=19da7f049b4f7f7286faf292c5419c37cc1e1768c2d8fe36b257ab0cf9c18874 MATRIX_SHA256=7613bcb25d32e61fac114da13af81e38354ec22a012a6deeaa168ee4cc454f87 DESIRED_MATRIX_SHA256=7613bcb25d32e61fac114da13af81e38354ec22a012a6deeaa168ee4cc454f87 MATRIX_STATUS=OK JOBS_BAD=0 FAILED_UNITS=0 BACKUP_RUN=NO RESTORE_RUN=NO CLOUD_WRITE_RUN=NO RETENTION_RUN=NO ROLLBACK=/var/lib/homelab-incidents/backup-matrix-desired-state-persistence-20260713T174036Z -HEALTH_FILE=alertmanager-two-cloud-backup.txt|mtime=2026-07-10 21:46:36.960914147 +0300|sha256=d90562fac18f643e765e60e54c673839c4ea57e580896241beb0ecc0ea45dea2 -STATUS=OK TS=2026-07-10T18:46:36Z TYPE=alertmanager-two-cloud-backup BAD=0 JOB=homelab-pve01-app-alertmanager-20260710T184605Z ARCHIVE_SIZE_BYTES=1107 LOCAL_STAGING_DELETED=true OFFHOST=OK RESTORE=OK -HEALTH_FILE=netbird-vps-trust-closure.txt|mtime=2026-07-07 18:29:49.166816011 +0300|sha256=6db8ee1ffbd2c371959917cac5ded56af3ec1829b01bd9c190e3334d6593a4a0 -STATUS=OK TS=2026-07-07T15:29:49Z TYPE=netbird-vps-trust-closure BAD=0 WARN=0 MODE=TRUST_CLOSURE_NO_KNOWN_HOSTS_CHANGE TRUSTED_NETBIRD_IDENTITY=edge-vm TRUSTED_NETBIRD_IP=[IP] PUBLIC_VPS_IP=[IP] PUBLIC_VPS_DECISION=REJECTED_HOSTKEY_MISMATCH PUBLIC_IP_STRICT_BLOCKED=YES PUBLIC_IP_KEYSCAN_MATCH=NO EDGE_REF_OK=1 NETBIRD_PEER_OK=1 REPORT=/var/lib/homelab-netbird-vps-trust/closure-latest.md -HEALTH_FILE=netbird-vps-trust-clean-seal.txt|mtime=2026-07-07 18:55:43.910666513 +0300|sha256=092eaa2c4f3a5f623e9440135d36e6bae7d8ae46a03d380d14a5e95657812b61 -HEALTH_FILE=searxng-backup.txt|mtime=2026-07-09 12:39:46.404327448 +0300|sha256=c1b03d9b8fee369c584a2c8f725639ed3ca77701d05dbe3b46fe5abb0f37a6b1 -STATUS=OK TS=2026-07-09T09:39:15Z TYPE=searxng-backup JOB=homelab-pve01-app-searxng-20260709T093915Z SOURCE=searxng_stack_config_cache SIZE=738 SHA256=352c9c54437b8ceabc491452d1d7150624eda8c4bf99c28dc78fd488ec06f28c LOCAL_BACKUP_DELETED=true NOTE=service-readiness-backup-proof -HEALTH_FILE=it-tools-backup.txt|mtime=2026-07-09 22:13:14.698027428 +0300|sha256=1481a6d2014b4caf49b7258d1922b575fadb4645a1a1878457c1f8f3eb8ca525 -STATUS=OK TS=2026-07-09T19:13:00Z TYPE=it-tools-backup BAD=0 APP=it-tools JOB=homelab-pve01-app-it-tools-20260709T191300Z SIZE=689 LOCAL_BACKUP_DELETED=true ENGINE=files JSON=/var/lib/homelab-backup/apps/latest-it-tools.json -HEALTH_FILE=netbird-vps-offhost.txt|mtime=2026-07-17 04:59:42.647642040 +0300|sha256=04ddca2c2a89ab344726695210540c4ca1358033f2edd22e2c97f1ab0aff4e0b -STATUS=OK -SERVICE=netbird-vps-offhost -TIMESTAMP_UTC=20260717T015941Z -DST_HOST=[IP] -DST=/mnt/staging/offhost/netbird-vps-backups-from-pve01 -LOCAL_LATEST=20260701T013054Z -REMOTE_LATEST=20260701T013054Z -REMOTE_FILES=30 -REMOTE_SIZE=466M -HEALTH_FILE=backup-scheduler-fairness-granularity.json|mtime=2026-08-10 20:50:58.416755115 +0300|sha256=5ae14f32dd777dbf4c6dee305db5767e69b9d47edac900d014ad243d8b108fbb -{ - "status": "OK", - "root_cause_count": 0, - "production_changed": true, - "cloud_changed": false, - "backup_started": false, - "applied_at_utc": "2026-08-10T17:50:58Z", - "target_bytes": true, - "global_dispatcher": true, - "fairness_changed": true, - "granularity_changed": true, - "one_heavy_target_per_decision": true, - "heavy_lock_released_between_targets": true, - "no_progress_guard": true, - "frozen_dependencies": true, - "systemd_changed": false, - "no_immediate_service_start": true, - "targets": { - "vm": "b0ca03aff1635a134b1638bf842df65e6fa26d965001b05eeb3ce68bf99811e5", - "app": "2c52087ea5f25c2763b9126120babdaf3535afff2203bab388ba3a2372a68819", - "dispatcher": "e83bf96c2e8290f6099e9e63033641fffd983947212ea3392d6197556a199849" - } -} -HEALTH_FILE=p2-media-backup.txt|mtime=2026-07-07 03:57:33.000124656 +0300|sha256=dccd8a9b1c9ae449783cbb8dc16ef9f2ead867b0fec20992722ec5f0aa9a4032 -HEALTH_FILE=actual-budget-backup.txt|mtime=2026-07-09 15:53:45.248005185 +0300|sha256=72d1b31e22e22ac91af66e3d8ae46b20b3b9297809ace524eece22031458fb05 -STATUS=OK TS=2026-07-09T12:53:43Z TYPE=actual-budget-backup JOB=homelab-pve01-app-actual-budget-20260709T051713Z SOURCE=latest-appbackup SIZE=2011 LOCAL_BACKUP_DELETED=true JSON=/var/lib/homelab-backup/apps/latest-actual-budget.json NOTE=service-readiness-backup-proof -HEALTH_FILE=backup-retention-safe-gc.json|mtime=2026-09-23 03:25:17.780212301 +0300|sha256=d5b48a3782e4181807fa6191ff5de57a1586afc151ec4cc7ba905f6764c03517 -{ - "schema": "homelab-backup-retention-v1", - "status": "OK", - "task_id": "backup-retention-safe-gc-v1", - "change_id": "CR-2026-0033", - "updated_at_utc": "2026-09-23T00:25:17Z", - "policy": { - "vm_keep": 2, - "app_keep": 7, - "control_keep": 4, - "min_delete_age_seconds": 86400, - "gfs": { - "enabled": true, - "critical_vm_count": 8, - "weekly_completed_buckets": 4, - "monthly_completed_buckets": 6, - "completed_periods_only": true - } - }, - "owner_aware_gc_proven": true, - "destructive_canary_proven": true, - "historical_gc_proven": false, - "last_plan": { - "delete_candidates": 4, - "protected": 0, - "kept": 139, - "gfs_kept": 28 - }, - "last_run_deleted": 4, - "legacy_namespaces_protected": [ - "hb", - "pve01", - "control-bootstrap" - ], - "provider_worm_proven": false -} -HEALTH_FILE=backup-matrix-cluster-desired-state-sync.txt|mtime=2026-07-13 20:51:32.416338448 +0300|sha256=c2530de4aaee5f4c2746d4e97b7e6d9ab70c3f4059afc9bacc75fd5e1980d659 -STATUS=OK TS=2026-07-13T17:51:31Z TYPE=backup-matrix-cluster-desired-state-sync BAD=0 PRE_SYNC_HEAD=af26ec3c4829b1330d49656205ac580ce1706263 GENERATED_SYNC_HEAD=597119391e7b265b85fd9a76644e4e33af47e469 GENERATED_SYNC_MODE=NEW_GENERATED_COMMIT GENERATED_SYNC_COMMIT_DELTA=1 MATRIX_STATUS=OK JOBS_TOTAL=35 JOBS_OK=31 JOBS_BAD=0 JOBS_UNMAPPED=0 JOBS_DISABLED=4 CLUSTER_PASSPORT=OK FINAL_READINESS=OK OVERALL=OK CLUSTER_ADMIN=OK PROBE_OK=4 PROBE_BAD=0 FAILED_UNITS=0 DESIRED_STATE_DIRTY=0 MATRIX_SHA256=7613bcb25d32e61fac114da13af81e38354ec22a012a6deeaa168ee4cc454f87 MATRIX_STATE_SHA256=14a31112aac5d577ae01df567921b093b2fedcf7189c4cc8f009216ee0749380 PRESEAL_SHA256=7ea1f710cd311bd51dba05a313b4c540f57d97407f008881e502fc213b4b3c99 BACKUP_RUN=NO RESTORE_RUN=NO CLOUD_WRITE_RUN=NO RETENTION_RUN=NO EVIDENCE_ROOT=/var/lib/homelab-change-sets/backup-matrix-cluster-final-seal/20260713T175131Z -HEALTH_FILE=control-backup.json|mtime=2026-09-23 11:26:22.485179218 +0300|sha256=0755695105c316d7ee6f3926a1ded277c1c878f92e0691e62eed98f90836dc0b -{ - "schema": "homelab-control-backup-health-v1", - "status": "OK", - "generation": "control-bootstrap-v2-pve01-20260916T120057Z", - "state_file": "/var/lib/homelab-backup/cloud-quorum/control-bootstrap-v2/control-bootstrap-v2-pve01-20260916T120057Z.json", - "restore_evidence": "/var/lib/homelab-backup/control-restore-validation/control-bootstrap/control-bootstrap-v2-pve01-20260916T120057Z.json", - "archive_sha256": "6fa394732cc1cafbf311a022b5fbdb6cf41a6562d9b3503666525c9392503e29", - "cloud_quorum": 2, - "control_freshness": "FRESH", - "full_site_dr_proven": false, - "updated_at_utc": "2026-09-23T08:26:22Z" -} -HEALTH_FILE=homelab-backup-baseline.txt|mtime=2026-06-17 23:02:13.952559036 +0300|sha256=4959757d29bb503315c60468dc0eb080b43cefde23d48e8ab192f6bd93b564d3 -STATUS=OK -SERVICE=homelab-backup-baseline -TIMESTAMP_UTC=20260617T200213Z -ENDPOINT_TOTAL=17 -ENDPOINT_OK=17 -ENDPOINT_BAD=0 -KUMA_TOTAL=18 -KUMA_UP=18 -KUMA_BAD=0 -ENDPOINT_SUMMARY=disk-space.txt:OK;immich-media-backup.txt:OK;immich-media-offhost.txt:OK;memos-backup.txt:OK;memos-offhost.txt:OK;paperless-backup.txt:OK;paperless-offhost.txt:OK;netbird-vps-backup.txt:OK;netbird-vps-offhost.txt:OK;netbird-vps-restore.txt:OK;netbird-peers.txt:OK;filebrowser-backup.txt:OK;filebrowser-offhost.txt:OK;filebrowser-restore.txt:OK;app-stacks-batch-backup.txt:OK;app-stacks-batch-offhost.txt:OK;app-stacks-batch-restore.txt:OK; -KUMA_MONITORS=1|Homelab Backup Health|active=1|user_id=1|latest_status=1|url=http://[IP]:9101/backup.txt;24|Immich Media Backup|active=1|user_id=1|latest_status=1|url=http://[IP]:9101/immich-media-backup.txt;25|Immich Media Offhost Backup|active=1|user_id=1|latest_status=1|url=http://[IP]:9101/immich-media-offhost.txt;26|Disk Space Health|active=1|user_id=1|latest_status=1|url=http://[IP]:9101/disk-space.txt;27|Memos Backup|active=1|user_id=1|latest_status=1|url=http://[IP]:9101/memos-backup.txt;28|Memos Offhost Backup|active=1|user_id=1|latest_status=1|url=http://[IP]:9101/memos-offhost.txt;29|Paperless Backup|active=1|user_id=1|latest_status=1|url=http://[IP]:9101/paperless-backup.txt;30|Paperless Offhost Backup|active=1|user_id=1|latest_status=1|url=http://[IP]:9101/paperless-offhost.txt;31|NetBird VPS Backup|active=1|user_id=1|latest_status=1|url=http://[IP]:9101/netbird-vps-backup.txt;32|NetBird VPS Offhost Backup|active=1|user_id=1|latest_status=1|url=http://[IP]:9101/netbird-vps-offhost.txt;33|NetBird VPS Restore Validation|active=1|user_id=1|latest_status=1|url=http://[IP]:9101/netbird-vps-restore.txt;34|NetBird Homelab Peers|active=1|user_id=1|latest_status=1|url=http://[IP]:9101/netbird-peers.txt;35|File Browser Backup|active=1|user_id=1|latest_status=1|url=http://[IP]:9101/filebrowser-backup.txt;36|File Browser Offhost Backup|active=1|user_id=1|latest_status=1|url=http://[IP]:9101/filebrowser-offhost.txt;37|File Browser Restore Validation|active=1|user_id=1|latest_status=1|url=http://[IP]:9101/filebrowser-restore.txt;38|App Stacks Batch Backup|active=1|user_id=1|latest_status=1|url=http://[IP]:9101/app-stacks-batch-backup.txt;39|App Stacks Batch Offhost|active=1|user_id=1|latest_status=1|url=http://[IP]:9101/app-stacks-batch-offhost.txt;40|App Stacks Batch Restore|active=1|user_id=1|latest_status=1|url=http://[IP]:9101/app-stacks-batch-restore.txt; -NOTES=none -HEALTH_FILE=memos-backup.txt|mtime=2026-08-30 21:08:09.015289529 +0300|sha256=01edf9ba034eb56e7a2ae8abd4ff6ee677fd19c320b4aedefe85d574fc46ee35 -STATUS=FAIL -TS=2026-08-30T18:08:08Z -TYPE=backup-matrix-generated -JOB_ID=memos_backup -JOB_NAME=memos-backup -MATRIX_MODE=authoritative -TIMER=memos-backup.timer -SERVICE=memos-backup.service -TIMER_ENABLED=not-found -TIMER_ACTIVE=inactive -SOURCE_KIND=health_file -EVIDENCE_LEVEL=artifact_or_health -SERVICE_RESULT=success -SERVICE_EXEC_MAIN_STATUS=0 -SOURCE_HEALTH=/var/lib/homelab-backup/apps/latest-memos.json -SOURCE_STATUS=OK -SOURCE_MTIME_UTC=2026-08-06T21:26:13Z -AGE_SECONDS=2061715 -MAX_AGE_SECONDS=259200 -FRESHNESS=STALE -LEGACY_SOURCE_PATH=/var/lib/homelab-health/memos-backup.txt -CANONICAL_JOB_ID=memos_backup -CANONICAL_STATE=/var/lib/homelab-backup-matrix/state.json -HEALTH_FILE=netbird-vps-backup.txt|mtime=2026-07-01 04:31:05.433646905 +0300|sha256=372804695d6dba94e29e254f2175701e976692e7b15ee83fdcb6f10e05ca624e -STATUS=OK -SERVICE=netbird-vps-backup -TIMESTAMP_UTC=20260701T013054Z -VPS_IP=[IP] -SNAPSHOT=/mnt/staging/netbird-vps-backups/snapshots/20260701T013054Z -LATEST_SYMLINK=/mnt/staging/netbird-vps-backups/snapshots/latest -FILES=2 -SIZE=32M -SHA256=0226dd820cb9ede0ea9e8ea4af6cef656ffa54cfb819048c222a88f4c14d8db9 -HEALTH_FILE=backup-matrix-cluster-preseal-convergence.txt|mtime=2026-07-13 20:48:14.631325305 +0300|sha256=7ea1f710cd311bd51dba05a313b4c540f57d97407f008881e502fc213b4b3c99 -STATUS=OK TS=2026-07-13T17:48:14Z TYPE=backup-matrix-cluster-preseal-convergence BAD=0 MATRIX_STATUS=OK JOBS_TOTAL=35 JOBS_BAD=0 JOBS_UNMAPPED=0 JOBS_DISABLED=4 CLUSTER_PASSPORT=OK FINAL_READINESS=OK OVERALL=OK CLUSTER_ADMIN=OK PROBE_OK=4 PROBE_BAD=0 WEBPANEL=OK FAILED_UNITS=0 DESIRED_STATE_HEAD=af26ec3c4829b1330d49656205ac580ce1706263 DESIRED_STATE_DIRTY=0 MATRIX_SHA256=7613bcb25d32e61fac114da13af81e38354ec22a012a6deeaa168ee4cc454f87 MATRIX_STATE_SHA256=3791fa78cdfb56cd7027f89350dff245abcffc4f935adfc336ceeea00854ac89 CLUSTER_PASSPORT_SHA256=1b8b924ae0e5ce6872fe5671318094ce494806c2db7efe28299b5ff49adc0d0d FINAL_READINESS_SHA256=ac5159fa8c497afe021b5cb232f08cfec0e1a4d516d8aae966ba03f7034f1412 OVERALL_SHA256=6ae529bfdf5811f686d3b53978f453ce4f8757c74fc1be105de219b13a61ba46 BACKUP_RUN=NO RESTORE_RUN=NO CLOUD_WRITE_RUN=NO RETENTION_RUN=NO EVIDENCE_ROOT=/var/lib/homelab-change-sets/backup-matrix-cluster-preseal-resume/20260713T174748Z -HEALTH_FILE=cluster-admin-stage4f-preapply-backup.txt|mtime=2026-07-14 10:54:50.681548259 +0300|sha256=cc242d1b6e2ef6b6d25f283c7d967bd4985222783ce65050d75e6e61230067ae -STATUS=OK TS=2026-07-14T07:54:50Z TYPE=cluster-admin-stage4f-preapply-backup-final-seal BAD=0 STAGE4F_CLOSED=YES NEXT_STAGE=4G_PRODUCTION_APPLY_READ_ONLY_ONLY PRODUCTION_CHANGED=NO DATABASE_CHANGED=NO APPLICATION_CHANGED=NO BACKUP_JOB=homelab-pve01-app-cluster-admin-incident-engine-20260714T064327Z ARCHIVE_SIZE_BYTES=2651545 DATABASE_DUMP_SIZE_BYTES=2814735 CURRENT_JOB_VERIFY_MARKERS=2 RESTORE_LIST_CHECK=OK PRODUCTION_STATE=0|0|OK DESIRED_STATE_HEAD=69be75caeecaa242567d2d06c735ac5db3a9c048 DESIRED_STATE_CLEAN=YES REPORT=/var/lib/homelab-cluster-admin-stage4/stage4f-preapply-backup-final-seal-v1/report.txt REPORT_SHA256=9ee244085b9d191370371560690ea87e7bbfe2f14226a06a4e8f6720b3a7439b -HEALTH_FILE=jellyfin-backup.txt|mtime=2026-07-09 21:16:17.662793576 +0300|sha256=23afe6670bf9250dada540261513ffff2c36ee33963347e5673699d4a10f77a2 -STATUS=OK TS=2026-07-09T18:16:16Z TYPE=jellyfin-backup BAD=0 APP=jellyfin JOB=homelab-pve01-app-jellyfin-20260709T051713Z SIZE=25920 LOCAL_BACKUP_DELETED=true JSON=/var/lib/homelab-backup/apps/latest-jellyfin.json -HEALTH_FILE=vikunja-backup.txt|mtime=2026-07-09 20:51:20.850795887 +0300|sha256=48bc526e9c24220ad6c15554b85b819d8b6ba35d854ab87da2324b1534e194c6 -STATUS=OK TS=2026-07-09T17:51:14Z TYPE=vikunja-backup BAD=0 APP=vikunja JOB=homelab-pve01-app-vikunja-20260709T051713Z SIZE=34831 LOCAL_BACKUP_DELETED=true JSON=/var/lib/homelab-backup/apps/latest-vikunja.json -HEALTH_FILE=hapusya-site-v2-preapply-backup.txt|mtime=2026-07-13 14:34:59.208365351 +0300|sha256=506879fe26ddf3c392c4d485d1db145cde846336cd626096555e2021ab2705af -STATUS=OK TS=2026-07-13T11:34:59Z TYPE=hapusya-site-v2-preapply-backup BAD=0 NPMPLUS_JOB=homelab-pve01-app-npmplus-edge-20260713T113435Z TWO_CLOUDS=OK RESTORE_CHECK=OK ROLLBACK=/var/lib/homelab-incidents/hapusya-site-v2-preapply-20260713T113458Z ROLLBACK_MANIFEST_SHA256=144506c766fe9fd3152e4edf94a8867895b3dc54f29faf1e329773922b8c5640 CANDIDATE_TREE_SHA256=0eb098f91ea706c884d36a8ce806968e82e715ded66bb96f61c2ed8106faca93 CANDIDATE_BROWSER=OK BROWSER_RUNS=5 VIDEO_DESKTOP=OK VIDEO_MOBILE=OK STATIC_FALLBACK=OK PRODUCTION_CHANGED=NO -HEALTH_FILE=ntfy-backup.txt|mtime=2026-07-09 23:39:41.327493875 +0300|sha256=e28d674bd0ba4c088c6c9f523da493e3674750837ab417223f7405ea61d8b020 -STATUS=OK TS=2026-07-09T20:39:29Z TYPE=ntfy-backup BAD=0 APP=ntfy JOB=homelab-pve01-app-ntfy-20260709T051713Z SIZE=13516 LOCAL_BACKUP_DELETED=true JSON=/var/lib/homelab-backup/apps/latest-ntfy.json -HEALTH_FILE=immich-media-backup.txt|mtime=2026-08-30 21:08:08.994511799 +0300|sha256=f94d804693a2ca18e51c66ade993a95c9b8ae183730e31013e3886010b03ff63 -STATUS=FAIL -TS=2026-08-30T18:08:08Z -TYPE=backup-matrix-generated -JOB_ID=immich_media_backup -JOB_NAME=immich-media-offhost-sync -MATRIX_MODE=authoritative -TIMER=immich-media-offhost-sync.timer -SERVICE=immich-media-offhost-sync.service -TIMER_ENABLED=not-found -TIMER_ACTIVE=inactive -SOURCE_KIND=health_file -EVIDENCE_LEVEL=artifact_or_health -SERVICE_RESULT=success -SERVICE_EXEC_MAIN_STATUS=0 -SOURCE_HEALTH=/var/lib/homelab-health/immich-media-offhost.txt -SOURCE_STATUS=OK -SOURCE_MTIME_UTC=2026-08-08T05:52:15Z -AGE_SECONDS=1944953 -MAX_AGE_SECONDS=259200 -FRESHNESS=STALE -LEGACY_SOURCE_PATH=/var/lib/homelab-health/immich-media-backup.txt -CANONICAL_JOB_ID=immich_media_backup -CANONICAL_STATE=/var/lib/homelab-backup-matrix/state.json -HEALTH_FILE=paperless-backup.txt|mtime=2026-07-18 01:52:29.893166209 +0300|sha256=8ae7e31227e48c86412eb9e49fa91ce3cbb97454ea3414e25d10ef5432e6cd46 -STATUS=OK -TS=2026-07-17T22:52:29Z -TYPE=backup-matrix-generated -JOB_ID=paperless_backup -JOB_NAME=paperless-backup -MATRIX_MODE=authoritative -TIMER=paperless-backup.timer -SERVICE=paperless-backup.service -TIMER_ENABLED=enabled -TIMER_ACTIVE=active -SOURCE_KIND=health_file -EVIDENCE_LEVEL=artifact_or_health -SERVICE_RESULT=success -SERVICE_EXEC_MAIN_STATUS=0 -SOURCE_HEALTH=/var/lib/homelab-backup/apps/latest-paperless.json -SOURCE_STATUS=OK -SOURCE_MTIME_UTC=2026-07-17T02:30:25Z -AGE_SECONDS=73324 -MAX_AGE_SECONDS=259200 -FRESHNESS=FRESH -LEGACY_SOURCE_PATH=/var/lib/homelab-health/paperless-backup.txt -CANONICAL_JOB_ID=paperless_backup -CANONICAL_STATE=/var/lib/homelab-backup-matrix/state.json -HEALTH_FILE=cluster-admin-stage1-preapply-backup.txt|mtime=2026-07-13 03:11:06.700511934 +0300|sha256=a70d7bba3c2b50deaa117e56862659b857e72682ec829ed7724b03dc92111105 -STATUS=OK TS=2026-07-13T00:10:57Z TYPE=cluster-admin-stage1-preapply-backup BAD=0 PRODUCTION_CHANGED=BACKUP_ARTIFACTS_ONLY PRODUCTION_DATABASE_CHANGED=NO SERVICE_RESTARTED=NO CLOUD_WRITE_RUN=NO ROLLBACK_POINT=VERIFIED DIRECT_DB_SNAPSHOT=VERIFIED DIRECT_DB_DUMP_SIZE_BYTES=1710825 DIRECT_DB_DUMP_SHA256=c0f13cec235ab3242eb8da4cad12962df1a2e37b25792a8ce9944c1d7de4312a APPBACKUP_JOB=homelab-pve01-app-cluster-admin-incident-engine-20260713T000543Z APPBACKUP_STATUS=success APPBACKUP_RESTORE_CHECK=OK MIGRATION_DRY_RUN=OK IDEMPOTENCE_TEST=OK ROLLBACK_TEST=OK COLLECTOR_COMPATIBILITY_TRIGGER=OK PRODUCTION_SCHEMA_UNCHANGED=YES READY_FOR_PRODUCTION_APPLY=YES DESIRED_STATE_HEAD=1379b4fb4e809e56e9bc16f0e0b19b0a509192d5 PREAPPLY_ROOT=/var/lib/homelab-stage1-preapply/20260713T000041Z REMOTE_SNAPSHOT_DIR=/var/lib/cluster-admin-preapply/20260713T000041Z EVIDENCE_MANIFEST=/var/lib/homelab-stage1-preapply/20260713T000041Z/SHA256SUMS -HEALTH_FILE=cadvisor-backup.txt|mtime=2026-07-09 11:25:57.266067724 +0300|sha256=cb0bcacf449ab29e8954014fb689d1022527f0b67aae0d4f0c808358cae55e07 -STATUS=OK TS=2026-07-09T08:25:56Z TYPE=cadvisor-backup JOB=homelab-pve01-app-cadvisor-20260709T082556Z SOURCE=cadvisor_stack_proxy SIZE=1384 SHA256=33e419912291daab19f8039dfe8971f277be0fe577eb3d0e538263bdf918eac8 LOCAL_BACKUP_DELETED=true NOTE=service-readiness-backup-proof -HEALTH_FILE=dockge-stacks-backup.txt|mtime=2026-07-09 13:12:46.609379653 +0300|sha256=f20ebfd8e230858d31286670ccb6a4699d4251209765fb37e3ff4309435113f5 -STATUS=OK TS=2026-07-09T10:12:44Z TYPE=dockge-stacks-backup JOB=homelab-pve01-app-dockge-stacks-20260709T072218Z SOURCE=latest-appbackup SIZE=134913 LOCAL_BACKUP_DELETED=true JSON=/var/lib/homelab-backup/apps/latest-dockge-stacks.json NOTE=service-readiness-backup-proof -HEALTH_FILE=dockge-backup.txt|mtime=2026-07-09 13:12:46.609379653 +0300|sha256=3f5198e9d0eb7852e4ee563109c801be03be268b2b10afeca68fbc39be4d35f7 -STATUS=OK TS=2026-07-09T10:12:44Z TYPE=dockge-backup JOB=homelab-pve01-app-dockge-stacks-20260709T072218Z SOURCE=appbackup-dockge-stacks SIZE=134913 LOCAL_BACKUP_DELETED=true JSON=/var/lib/homelab-backup/apps/latest-dockge.json NOTE=service-readiness-backup-proof -HEALTH_FILE=backup-lock-fd-hardening.json|mtime=2026-08-10 17:49:25.372646496 +0300|sha256=d64d18259e58525a9549f6d2b40b331ea6cc7b7057fd17050aebc585d73731a5 -{ - "status": "OK", - "root_cause_count": 0, - "production_changed": true, - "cloud_changed": false, - "backup_started": false, - "applied_at_utc": "2026-08-10T14:49:25Z", - "target_bytes": true, - "fd_child_boundary": true, - "parent_lock_ownership_preserved": true, - "systemd_changed": false, - "no_immediate_service_start": true, - "targets": { - "vm": "55c2c30d93590ce0367667c2de4cf8e6436bd10a2a35fb7b775f918ba9b52ffb", - "app": "adcd592c619c96c99be19777eff98bbf0226e878a9635c718e9113b662960a5a" - } -} -HEALTH_FILE=backup-sla.txt|mtime=2026-08-30 21:08:08.601604759 +0300|sha256=4762bb7cfb98ab1311adbd1e80287f7e18f79983656a888a056ed730ecdf0372 -STATUS=FAIL -TS=2026-08-30T18:08:08Z -TYPE=backup-matrix-generated -JOB_ID=homelab_backup_sla_health -JOB_NAME=homelab-backup-sla-health -MATRIX_MODE=authoritative -TIMER=homelab-backup-sla-health.timer -SERVICE=homelab-backup-sla-health.service -TIMER_ENABLED=not-found -TIMER_ACTIVE=inactive -SOURCE_KIND=health_file -EVIDENCE_LEVEL=artifact_or_health -SERVICE_RESULT=success -SERVICE_EXEC_MAIN_STATUS=0 -SOURCE_HEALTH=/var/lib/homelab-backup-matrix/evidence/homelab_backup_sla_health/0-backup-sla.txt -SOURCE_STATUS=WARN -SOURCE_MTIME_UTC=2026-08-06T21:34:03Z -AGE_SECONDS=2061245 -MAX_AGE_SECONDS=172800 -FRESHNESS=STALE -LEGACY_SOURCE_PATH=/var/lib/homelab-health/backup-sla.txt -CANONICAL_JOB_ID=homelab_backup_sla_health -CANONICAL_STATE=/var/lib/homelab-backup-matrix/state.json -HEALTH_FILE=deep-service-backup-audit.txt|mtime=2026-07-10 00:12:45.326483756 +0300|sha256=16dfd49e8f0618a836c473874b7a70fa896771dff5e2613e510e6d0a02eca7b4 -STATUS=OK TS=2026-07-09T21:12:44Z TYPE=deep-service-backup-audit BAD=0 SERVICE_READINESS_OK_COUNT=47 NON_OK=0 APPBACKUP_FLAG_GAPS=0 CONTAINER_GAPS=0 PROM_FIRING=0 BACKUP_SLA=OK ALERTING=OK EXTERNAL_LINKS_CLASSIFIED=OK SERVICES_REMAINING_TOTAL=0 BACKUP_GAPS_TOTAL=0 COMMANDS_REMAINING_TOTAL=0 -HEALTH_FILES_END=1 -DEPENDENCY_FILE_INDEX_BEGIN=1 -DEPENDENCY_FILE_COUNT=34 -DEPENDENCY_FILE|path=/etc/homelab-backup-v2/targets.json|mtime=2026-09-22 12:19:09.941039348 +0300|sha256=decea6b876c8ede15c91cad69c8e15e46021e5e96d25d415139bf2c524e23f7b -2: "created_by": "SUPPORT-260914-INFRA-DR-BACKUPV2-SHADOW-PERSISTENT-PRODUCERS-STAGE-GREEN1-389R1", -169: "logical_id": "md-relay", -193: "logical_id": "us-netbird", -DEPENDENCY_FILE|path=/etc/systemd/system/netbird-vps-backup.service.d/10-superseded-noop.conf|mtime=2026-07-08 08:25:22.050610124 +0300|sha256=4cf3f2825b770b32447624a611bce605d97987bd2c6e8862c84d35b166f9102a -3:ExecStart=/usr/local/sbin/homelab-superseded-unit-ok netbird-vps-backup.service superseded_by_current_appbackup_or_trust_proof -DEPENDENCY_FILE|path=/root/771ad_final_crowdsec_capi_netbird_closure.sh|mtime=2026-07-02 22:10:57.781648619 +0300|sha256=c99d08385f5d7c0a266c1c8002b3c7b054e1bb561eb48b1f8a227f37fe430ab6 -8:PROOF="/root/evidence/771AD_FINAL_CROWDSEC_CAPI_NETBIRD_CLOSURE_${TS}_PROOF.txt" -24: echo "STEP=771AD_FINAL_CROWDSEC_CAPI_NETBIRD_CLOSURE" -47: echo NETBIRD_EXIT_ROUTE_VERIFY -48: netbird status 2>&1 | redact || true -59: curl -4 -sk --http1.1 --connect-timeout 12 --max-time 45 -o /tmp/771ad_capi.body -w \"CAPI_HTTP=%{http_code}\\n\" https://api.crowdsec.net/v3/watchers/login || true -61: echo CROWDSEC_STATUS -62: docker ps --format \"{{.Names}} {{.Status}} {{.Ports}}\" | grep -E \"^(crowdsec|socket-proxy|npmplus)\" || true -63: docker exec crowdsec cscli lapi status 2>&1 | redact || true -64: docker exec crowdsec cscli capi status 2>&1 | redact || true -67: cd /opt/stacks/crowdsec -78: docker logs --since 10m crowdsec 2>&1 | grep -Ei \"fatal|TLS handshake timeout|unable to run local API|failed to get docker info|lookup socket-proxy|server misbehaving|mapping value is not allowed|missing credentials field|online_api_credentials\" | redact || echo RECENT_FATALS_NONE -80: echo HEALTH_FILE -81: cat /var/lib/homelab-health/crowdsec-capi.txt 2>/dev/null | redact || true -97: if ! grep -q 'CROWDSEC_CAPI_NETBIRD_EXIT_ROUTE_20260702' "$REF"; then -100:## CROWDSEC_CAPI_NETBIRD_EXIT_ROUTE_20260702 -101:- edge-vm CrowdSec CAPI is registered and enabled through NetBird exit route. -102:- Client peer: edge-vm.netbird.selfhosted / [IP]. -103:- Primary exit routing peer: relay.netbird.selfhosted / [IP] / Moldova. -104:- Backup egress peer present: mail.netbird.selfhosted / [IP] / USA. -105:- NetBird exit route proof on edge-vm: wg allowed-ips includes [IP]/0 via relay; ip route get [IP] uses wt0 table 7120. -107:- CrowdSec proof: cscli lapi status OK, cscli capi status OK, CAPI sharing/blocklist pull enabled, no DISABLE_ONLINE_API or -no-capi flags in active compose. -109:- Unauthenticated https://api.crowdsec.net/v3/watchers/login returning HTTP 403 is expected network reachability proof. -118: if ! grep -q 'LESSON_20260702_CROWDSEC_NETBIRD_EXIT_ROUTE' "$ERR"; then -121:## LESSON_20260702_CROWDSEC_NETBIRD_EXIT_ROUTE -122:- Do not treat e3qxxx.netbird.selfhosted / [IP] and e3qxxx-183-106.netbird.selfhosted / [IP] as VPS egress nodes; they are user mobile peers. -123:- Correct egress candidates are relay.pvepro.ru / relay.netbird.selfhosted / [IP] for Moldova and mail.pvepro.ru / mail.netbird.selfhosted / [IP] for USA. -124:- Access Policy edge-vm -> relay/mail only proves peer reachability. It is not an internet exit route. -125:- Required NetBird fix for CrowdSec CAPI was Network Routing exit route [IP]/0 distributed to edge-vm group, routing peer relay, masquerade/NAT enabled. -126:- Before attempting CrowdSec CAPI, verify on edge-vm: ip route get [IP] uses wt0, wg allowed-ips contains [IP]/0, and Cloudflare trace no longer shows home IP [IP]. -134: echo "STATUS=OK_771AD_FINAL_CROWDSEC_CAPI_NETBIRD_CLOSURE_DONE" -DEPENDENCY_FILE|path=/root/771ag_publish_final_health_index.sh|mtime=2026-07-02 22:30:48.251143514 +0300|sha256=8b7c814c3893ebb2068d90c29ec01460895aba7cba44d78ba8d7d2a8936c1b7a -8:PROOF="/root/evidence/771AG_PUBLISH_FINAL_HEALTH_INDEX_${TS}_PROOF.txt" -10:LOCAL_HEALTH="/var/lib/homelab-health/crowdsec-capi-netbird-final.txt" -23: echo "STEP=771AG_PUBLISH_FINAL_HEALTH_INDEX" -26: echo "CONFIG_CHANGE=NO_SERVICE_CHANGE_HEALTH_INDEX_ONLY" -31: grep -n 'CROWDSEC_CAPI_NETBIRD_EXIT_ROUTE_20260702' /etc/pve/31_HOMELAB_REFERENCE.md || true -32: grep -n 'LESSON_20260702_CROWDSEC_NETBIRD_EXIT_ROUTE' /etc/pve/HOMELAB_ASSISTANT_ERROR_REGISTER.md || true -34: echo "WRITE_LOCAL_HEALTH_SUMMARY" -35: mkdir -p /var/lib/homelab-health -36: cat >"$LOCAL_HEALTH" </dev/null | redact || true -77: echo EDGE_CROWDSEC -78: docker exec crowdsec cscli lapi status 2>&1 | redact || true -79: docker exec crowdsec cscli capi status 2>&1 | redact || true -82: docker inspect crowdsec --format \"RestartCount={{.RestartCount}}\" 2>/dev/null || true -83: cd /opt/stacks/crowdsec -93: for cmd in /usr/local/sbin/homelab-health-metrics /usr/local/sbin/homelab-evidence-catalog.sh; do -111: echo "LOCAL_HEALTH_FINAL" -112: cat "$LOCAL_HEALTH" -115: echo "STATUS=OK_771AG_PUBLISH_FINAL_HEALTH_INDEX_DONE" -DEPENDENCY_FILE|path=/root/771ak_final_done_marker.sh|mtime=2026-07-02 22:34:56.049721292 +0300|sha256=64f3b7e547623a9908eab9f09a1edcaeac8795af30db85e975dfa2e95cbe0d2a -15: echo "RESULT=CROWDSEC_CAPI_CLOSED" -18: echo "EXIT_ROUTE=relay.netbird.selfhosted/[IP]" -23: echo "REFERENCE_MARKER_LINE=$(grep -n 'CROWDSEC_CAPI_NETBIRD_EXIT_ROUTE_20260702' /etc/pve/31_HOMELAB_REFERENCE.md | cut -d: -f1 | head -1)" -24: echo "ERROR_REGISTER_MARKER_LINE=$(grep -n 'LESSON_20260702_CROWDSEC_NETBIRD_EXIT_ROUTE' /etc/pve/HOMELAB_ASSISTANT_ERROR_REGISTER.md | cut -d: -f1 | head -1)" -25: echo "FINAL_HEALTH=$(grep -m1 '^STATUS=' /var/lib/homelab-health/crowdsec-capi-netbird-final.txt 2>/dev/null || echo MISSING)" -26: echo "SOAK_HEALTH_ON_EDGE=$(ssh debian@[IP] 'sudo grep -m1 "^STATUS=" /var/lib/homelab-health/crowdsec-capi.txt 2>/dev/null || echo MISSING')" -28: ls -1 /root/evidence/771AC_REGISTER_CROWDSEC_CAPI_AFTER_EXIT_ROUTE_20260702T190428Z_PROOF.txt \ -29: /root/evidence/771AD_FINAL_CROWDSEC_CAPI_NETBIRD_CLOSURE_20260702T191057Z_PROOF.txt \ -30: /root/evidence/771AE_CROWDSEC_CAPI_SOAK_FINAL_20260702T191745Z_PROOF.txt \ -31: /root/evidence/771AF_CROWDSEC_CAPI_PROOF_INDEX_20260702T192904Z.txt \ -32: /root/evidence/771AG_PUBLISH_FINAL_HEALTH_INDEX_20260702T193048Z_PROOF.txt \ -DEPENDENCY_FILE|path=/root/771l_crowdsec_capi_netbird_vps_egress_remote_20260702T151045Z.sh|mtime=2026-07-02 18:10:45.300196691 +0300|sha256=6a6df3cc085363c2bbbb92b8c3083123fb0e583e14e242c583a321bdb5612002 -5:STACK=/opt/stacks/crowdsec -7:BACKUP_DIR="$STACK/manual-backups/771l-netbird-vps-egress-$TS" -8:HEALTH_DIR=/var/lib/homelab-health -9:HEALTH_FILE="$HEALTH_DIR/crowdsec-capi.txt" -10:EGRESS_DIR=/etc/homelab-crowdsec-capi-netbird-egress -11:EGRESS_SERVICE=homelab-crowdsec-capi-netbird-egress.service -14:mkdir -p "$BACKUP_DIR" "$HEALTH_DIR" "$EGRESS_DIR/profiles" -20:write_health() { -26: echo "CHECK=crowdsec-capi" -29: } > "$HEALTH_FILE" -33: docker network inspect crowdsec_default --format '{{(index .IPAM.Config 0).Gateway}}' 2>/dev/null || echo "[IP]" -41: crowdsec: -66: docker compose up -d --force-recreate crowdsec -74: state="$(docker inspect crowdsec --format 'Running={{.State.Running}} Restarting={{.State.Restarting}} RestartCount={{.RestartCount}} StartedAt={{.State.StartedAt}} ExitCode={{.State.ExitCode}}' 2>/dev/null || true)" -75: health="$(curl -sk --connect-timeout 3 --max-time 6 -o /tmp/771l_health.html -w '%{http_code}' http://[IP]:8088/health || true)" -78: fatal="$(docker logs --since 30s crowdsec 2>&1 | grep -Eic 'fatal|unable to run local API|failed to get docker info|lookup socket-proxy|server misbehaving|mapping value is not allowed' || true)" -79: echo "LAPI_TRY=$n STATE=$state HEALTH=$health LAPI_RC=${lapi_rc:-NA} FATAL_COUNT_30S=$fatal" -83: && test "$health" = "200" \ -126:remove("/opt/stacks/crowdsec/config/config.yaml") -127:remove("/opt/stacks/crowdsec/config/user.yaml") -134:p = Path("/opt/stacks/crowdsec/config/config.yaml") -165: " credentials_path: /etc/crowdsec/online_api_credentials.yaml", -177: crowdsec: -182: NO_PROXY: "[IP],localhost,[IP],[IPV6],socket-proxy,crowdsec" -221:collect_netbird_peers() { -222: echo "NETBIRD_STATUS" -223: if ! command -v netbird >/dev/null 2>&1; then -224: echo "NETBIRD_CLI_MISSING" -228: netbird status 2>&1 | redact | tee /tmp/771l_netbird_status.txt || true -229: netbird status --json >/tmp/771l_netbird_status.json 2>/tmp/771l_netbird_status_json.err || true -230: netbird ip 2>/tmp/771l_netbird_ip.err | tee /tmp/771l_netbird_ip.txt || true -232: echo "NETBIRD_INTERFACES" -233: ip -o -4 addr show | grep -Ei 'wt|netbird|wg|100\.' || true -235: python3 - <<'PY' >/tmp/771l_netbird_peers.tsv -240:for p in [Path("/tmp/771l_netbird_ip.txt")]: -244:text = Path("/tmp/771l_netbird_status.txt").read_text(errors="ignore") if Path("/tmp/771l_netbird_status.txt").exists() else "" -DEPENDENCY_FILE|path=/root/771l_crowdsec_capi_netbird_vps_egress.sh|mtime=2026-07-02 18:10:45.295356498 +0300|sha256=401396a25d4a5cfa487f67b355b45b27140ed1ac6b49b41e342306cb618dfdb5 -8:PROOF="/root/evidence/771L_CROWDSEC_CAPI_NETBIRD_VPS_EGRESS_${TS}_PROOF.txt" -10:REMOTE_SCRIPT="/tmp/771l_crowdsec_capi_netbird_vps_egress_${TS}.sh" -11:LOCAL_REMOTE_SCRIPT="/root/771l_crowdsec_capi_netbird_vps_egress_remote_${TS}.sh" -18:STACK=/opt/stacks/crowdsec -20:BACKUP_DIR="$STACK/manual-backups/771l-netbird-vps-egress-$TS" -21:HEALTH_DIR=/var/lib/homelab-health -22:HEALTH_FILE="$HEALTH_DIR/crowdsec-capi.txt" -23:EGRESS_DIR=/etc/homelab-crowdsec-capi-netbird-egress -24:EGRESS_SERVICE=homelab-crowdsec-capi-netbird-egress.service -27:mkdir -p "$BACKUP_DIR" "$HEALTH_DIR" "$EGRESS_DIR/profiles" -33:write_health() { -39: echo "CHECK=crowdsec-capi" -42: } > "$HEALTH_FILE" -46: docker network inspect crowdsec_default --format '{{(index .IPAM.Config 0).Gateway}}' 2>/dev/null || echo "[IP]" -54: crowdsec: -79: docker compose up -d --force-recreate crowdsec -87: state="$(docker inspect crowdsec --format 'Running={{.State.Running}} Restarting={{.State.Restarting}} RestartCount={{.RestartCount}} StartedAt={{.State.StartedAt}} ExitCode={{.State.ExitCode}}' 2>/dev/null || true)" -88: health="$(curl -sk --connect-timeout 3 --max-time 6 -o /tmp/771l_health.html -w '%{http_code}' http://[IP]:8088/health || true)" -91: fatal="$(docker logs --since 30s crowdsec 2>&1 | grep -Eic 'fatal|unable to run local API|failed to get docker info|lookup socket-proxy|server misbehaving|mapping value is not allowed' || true)" -92: echo "LAPI_TRY=$n STATE=$state HEALTH=$health LAPI_RC=${lapi_rc:-NA} FATAL_COUNT_30S=$fatal" -96: && test "$health" = "200" \ -139:remove("/opt/stacks/crowdsec/config/config.yaml") -140:remove("/opt/stacks/crowdsec/config/user.yaml") -147:p = Path("/opt/stacks/crowdsec/config/config.yaml") -178: " credentials_path: /etc/crowdsec/online_api_credentials.yaml", -190: crowdsec: -195: NO_PROXY: "[IP],localhost,[IP],[IPV6],socket-proxy,crowdsec" -234:collect_netbird_peers() { -235: echo "NETBIRD_STATUS" -236: if ! command -v netbird >/dev/null 2>&1; then -237: echo "NETBIRD_CLI_MISSING" -241: netbird status 2>&1 | redact | tee /tmp/771l_netbird_status.txt || true -242: netbird status --json >/tmp/771l_netbird_status.json 2>/tmp/771l_netbird_status_json.err || true -243: netbird ip 2>/tmp/771l_netbird_ip.err | tee /tmp/771l_netbird_ip.txt || true -245: echo "NETBIRD_INTERFACES" -246: ip -o -4 addr show | grep -Ei 'wt|netbird|wg|100\.' || true -DEPENDENCY_FILE|path=/root/771m_purge_warp_fix_netbird_egress.sh|mtime=2026-07-02 18:16:37.903239779 +0300|sha256=0ae3b0e1e8016da55cf1756e868c51a23fe12328fae6ef4baa61c50b9e48e715 -8:PROOF="/root/evidence/771M_PURGE_WARP_FIX_NETBIRD_EGRESS_${TS}_PROOF.txt" -10:REMOTE_SCRIPT="/tmp/771m_purge_warp_fix_netbird_${TS}.sh" -11:LOCAL_REMOTE_SCRIPT="/root/771m_purge_warp_fix_netbird_remote_${TS}.sh" -18:STACK=/opt/stacks/crowdsec -20:BACKUP_DIR="$STACK/manual-backups/771m-purge-warp-netbird-egress-$TS" -21:HEALTH_DIR=/var/lib/homelab-health -22:HEALTH_FILE="$HEALTH_DIR/crowdsec-capi.txt" -23:EGRESS_DIR=/etc/homelab-crowdsec-capi-netbird-egress -24:EGRESS_SERVICE=homelab-crowdsec-capi-netbird-egress.service -28:mkdir -p "$BACKUP_DIR" "$HEALTH_DIR" "$EGRESS_DIR/profiles" -34:write_health() { -40: echo "CHECK=crowdsec-capi" -43: } > "$HEALTH_FILE" -47: docker network inspect crowdsec_default --format '{{(index .IPAM.Config 0).Gateway}}' 2>/dev/null || echo "[IP]" -55: crowdsec: -80: docker compose up -d --force-recreate crowdsec -88: state="$(docker inspect crowdsec --format 'Running={{.State.Running}} Restarting={{.State.Restarting}} RestartCount={{.RestartCount}} StartedAt={{.State.StartedAt}} ExitCode={{.State.ExitCode}}' 2>/dev/null || true)" -89: health="$(curl -sk --connect-timeout 3 --max-time 6 -o /tmp/771m_health.html -w '%{http_code}' http://[IP]:8088/health || true)" -92: fatal="$(docker logs --since 30s crowdsec 2>&1 | grep -Eic 'fatal|unable to run local API|failed to get docker info|lookup socket-proxy|server misbehaving|mapping value is not allowed' || true)" -93: echo "LAPI_TRY=$n STATE=$state HEALTH=$health LAPI_RC=${lapi_rc:-NA} FATAL_COUNT_30S=$fatal" -97: && test "$health" = "200" \ -140:remove("/opt/stacks/crowdsec/config/config.yaml") -141:remove("/opt/stacks/crowdsec/config/user.yaml") -148:p = Path("/opt/stacks/crowdsec/config/config.yaml") -179: " credentials_path: /etc/crowdsec/online_api_credentials.yaml", -191: crowdsec: -196: NO_PROXY: "[IP],localhost,[IP],[IPV6],socket-proxy,crowdsec" -258:collect_netbird_peers() { -259: echo "NETBIRD_STATUS" -260: if ! command -v netbird >/dev/null 2>&1; then -261: echo "NETBIRD_CLI_MISSING" -265: netbird status 2>&1 | redact | tee /tmp/771m_netbird_status.txt || true -266: netbird status --json >/tmp/771m_netbird_status.json 2>/tmp/771m_netbird_status_json.err || true -267: netbird ip 2>/tmp/771m_netbird_ip.err | tee /tmp/771m_netbird_ip.txt || true -269: echo "NETBIRD_INTERFACES" -270: ip -o -4 addr show | grep -Ei 'wt|netbird|wg|100\.' || true -DEPENDENCY_FILE|path=/root/771m_purge_warp_fix_netbird_remote_20260702T151637Z.sh|mtime=2026-07-02 18:16:37.909445276 +0300|sha256=941f683d148a01d9af9ef6ce938c8ee6874d7af1ac1ac5ee96c7d9ca7cc262a4 -5:STACK=/opt/stacks/crowdsec -7:BACKUP_DIR="$STACK/manual-backups/771m-purge-warp-netbird-egress-$TS" -8:HEALTH_DIR=/var/lib/homelab-health -9:HEALTH_FILE="$HEALTH_DIR/crowdsec-capi.txt" -10:EGRESS_DIR=/etc/homelab-crowdsec-capi-netbird-egress -11:EGRESS_SERVICE=homelab-crowdsec-capi-netbird-egress.service -15:mkdir -p "$BACKUP_DIR" "$HEALTH_DIR" "$EGRESS_DIR/profiles" -21:write_health() { -27: echo "CHECK=crowdsec-capi" -30: } > "$HEALTH_FILE" -34: docker network inspect crowdsec_default --format '{{(index .IPAM.Config 0).Gateway}}' 2>/dev/null || echo "[IP]" -42: crowdsec: -67: docker compose up -d --force-recreate crowdsec -75: state="$(docker inspect crowdsec --format 'Running={{.State.Running}} Restarting={{.State.Restarting}} RestartCount={{.RestartCount}} StartedAt={{.State.StartedAt}} ExitCode={{.State.ExitCode}}' 2>/dev/null || true)" -76: health="$(curl -sk --connect-timeout 3 --max-time 6 -o /tmp/771m_health.html -w '%{http_code}' http://[IP]:8088/health || true)" -79: fatal="$(docker logs --since 30s crowdsec 2>&1 | grep -Eic 'fatal|unable to run local API|failed to get docker info|lookup socket-proxy|server misbehaving|mapping value is not allowed' || true)" -80: echo "LAPI_TRY=$n STATE=$state HEALTH=$health LAPI_RC=${lapi_rc:-NA} FATAL_COUNT_30S=$fatal" -84: && test "$health" = "200" \ -127:remove("/opt/stacks/crowdsec/config/config.yaml") -128:remove("/opt/stacks/crowdsec/config/user.yaml") -135:p = Path("/opt/stacks/crowdsec/config/config.yaml") -166: " credentials_path: /etc/crowdsec/online_api_credentials.yaml", -178: crowdsec: -183: NO_PROXY: "[IP],localhost,[IP],[IPV6],socket-proxy,crowdsec" -245:collect_netbird_peers() { -246: echo "NETBIRD_STATUS" -247: if ! command -v netbird >/dev/null 2>&1; then -248: echo "NETBIRD_CLI_MISSING" -252: netbird status 2>&1 | redact | tee /tmp/771m_netbird_status.txt || true -253: netbird status --json >/tmp/771m_netbird_status.json 2>/tmp/771m_netbird_status_json.err || true -254: netbird ip 2>/tmp/771m_netbird_ip.err | tee /tmp/771m_netbird_ip.txt || true -256: echo "NETBIRD_INTERFACES" -257: ip -o -4 addr show | grep -Ei 'wt|netbird|wg|100\.' || true -259: python3 - <<'PY' >/tmp/771m_netbird_peers.tsv -264:p = Path("/tmp/771m_netbird_ip.txt") -278:jp = Path("/tmp/771m_netbird_status.json") -DEPENDENCY_FILE|path=/root/771n_netbird_usa_moldova_egress_only.sh|mtime=2026-07-02 18:21:39.746368388 +0300|sha256=420d835318960ae2644dafc93e0427584505acaffa9786927623aeb59ba6ac0e -8:PROOF="/root/evidence/771N_NETBIRD_USA_MOLDOVA_EGRESS_ONLY_${TS}_PROOF.txt" -10:REMOTE_SCRIPT="/tmp/771n_netbird_usa_moldova_egress_${TS}.sh" -11:LOCAL_REMOTE_SCRIPT="/root/771n_netbird_usa_moldova_egress_remote_${TS}.sh" -18:STACK=/opt/stacks/crowdsec -20:EGRESS_DIR=/etc/homelab-crowdsec-capi-netbird-egress -21:HEALTH_DIR=/var/lib/homelab-health -22:HEALTH_FILE="$HEALTH_DIR/crowdsec-capi.txt" -25:mkdir -p "$EGRESS_DIR/profiles" "$HEALTH_DIR" -32: docker network inspect crowdsec_default --format '{{(index .IPAM.Config 0).Gateway}}' 2>/dev/null || echo "[IP]" -35:write_health() { -41: echo "CHECK=crowdsec-capi" -44: } > "$HEALTH_FILE" -47:force_crowdsec_safe() { -51: crowdsec: -74: docker compose up -d crowdsec -84: cat >"$EGRESS_DIR/profiles/${profile}.env" </etc/systemd/system/homelab-crowdsec-capi-netbird-egress.service <<'EOF' -95:Description=Homelab CrowdSec CAPI NetBird VPS SOCKS egress -96:After=network-online.target docker.service netbird.service -102:EnvironmentFile=/etc/homelab-crowdsec-capi-netbird-egress/current.env -111: cat >/usr/local/sbin/homelab-crowdsec-capi-egress-switch <<'EOF' -115:base=/etc/homelab-crowdsec-capi-netbird-egress -123:systemctl restart homelab-crowdsec-capi-netbird-egress.service -126:systemctl --no-pager --full status homelab-crowdsec-capi-netbird-egress.service | sed -n '1,30p' -128: chmod 755 /usr/local/sbin/homelab-crowdsec-capi-egress-switch -131: systemctl enable --now homelab-crowdsec-capi-netbird-egress.service -132: systemctl restart homelab-crowdsec-capi-netbird-egress.service -134: systemctl is-active homelab-crowdsec-capi-netbird-egress.service || true -146: if "HOMELAB_771" in line and "CROWDSEC_CAPI" in line and "BEGIN" in line: -149: if "HOMELAB_771" in line and "CROWDSEC_CAPI" in line and "END" in line: -160:out.append("# HOMELAB_771N_CROWDSEC_CAPI_NETBIRD_BEGIN") -163:out.append("# HOMELAB_771N_CROWDSEC_CAPI_NETBIRD_END") -194: code="$(curl -sk --proxy "socks5h://[IP]:${port}" --connect-timeout 12 --max-time 45 -o /tmp/771n_capi_${profile}.body -w '%{http_code}' https://api.crowdsec.net/v3/watchers/login || true)" -203: echo "VPS_EGRESS_OK profile=$profile user=$user ip=$ip" -214: echo "VPS_EGRESS_NOT_READY profile=$profile ip=$ip" -DEPENDENCY_FILE|path=/root/771n_netbird_usa_moldova_egress_remote_20260702T152139Z.sh|mtime=2026-07-02 18:21:39.750535800 +0300|sha256=1c25cbc13f524d0f6974a71c255c5634c69380818cb1b5ad4f8f6ea9bf8c6a01 -5:STACK=/opt/stacks/crowdsec -7:EGRESS_DIR=/etc/homelab-crowdsec-capi-netbird-egress -8:HEALTH_DIR=/var/lib/homelab-health -9:HEALTH_FILE="$HEALTH_DIR/crowdsec-capi.txt" -12:mkdir -p "$EGRESS_DIR/profiles" "$HEALTH_DIR" -19: docker network inspect crowdsec_default --format '{{(index .IPAM.Config 0).Gateway}}' 2>/dev/null || echo "[IP]" -22:write_health() { -28: echo "CHECK=crowdsec-capi" -31: } > "$HEALTH_FILE" -34:force_crowdsec_safe() { -38: crowdsec: -61: docker compose up -d crowdsec -71: cat >"$EGRESS_DIR/profiles/${profile}.env" </etc/systemd/system/homelab-crowdsec-capi-netbird-egress.service <<'EOF' -82:Description=Homelab CrowdSec CAPI NetBird VPS SOCKS egress -83:After=network-online.target docker.service netbird.service -89:EnvironmentFile=/etc/homelab-crowdsec-capi-netbird-egress/current.env -98: cat >/usr/local/sbin/homelab-crowdsec-capi-egress-switch <<'EOF' -102:base=/etc/homelab-crowdsec-capi-netbird-egress -110:systemctl restart homelab-crowdsec-capi-netbird-egress.service -113:systemctl --no-pager --full status homelab-crowdsec-capi-netbird-egress.service | sed -n '1,30p' -115: chmod 755 /usr/local/sbin/homelab-crowdsec-capi-egress-switch -118: systemctl enable --now homelab-crowdsec-capi-netbird-egress.service -119: systemctl restart homelab-crowdsec-capi-netbird-egress.service -121: systemctl is-active homelab-crowdsec-capi-netbird-egress.service || true -133: if "HOMELAB_771" in line and "CROWDSEC_CAPI" in line and "BEGIN" in line: -136: if "HOMELAB_771" in line and "CROWDSEC_CAPI" in line and "END" in line: -147:out.append("# HOMELAB_771N_CROWDSEC_CAPI_NETBIRD_BEGIN") -150:out.append("# HOMELAB_771N_CROWDSEC_CAPI_NETBIRD_END") -181: code="$(curl -sk --proxy "socks5h://[IP]:${port}" --connect-timeout 12 --max-time 45 -o /tmp/771n_capi_${profile}.body -w '%{http_code}' https://api.crowdsec.net/v3/watchers/login || true)" -190: echo "VPS_EGRESS_OK profile=$profile user=$user ip=$ip" -201: echo "VPS_EGRESS_NOT_READY profile=$profile ip=$ip" -206:echo "STEP=771N_NETBIRD_USA_MOLDOVA_EGRESS_ONLY" -207:echo "RULE=WARP_PURGED_NETBIRD_ONLY" -215:echo "CROWDSEC_SAFE_STATE" -DEPENDENCY_FILE|path=/root/771o_netbird_management_acl_discovery.sh|mtime=2026-07-02 18:24:39.066582039 +0300|sha256=50526afeaeb99fa9a8018374abe82731dcb9e93eac4be64934d0c6a8610be322 -8:PROOF="/root/evidence/771O_NETBIRD_MANAGEMENT_ACL_DISCOVERY_${TS}_PROOF.txt" -14: echo "STEP=771O_NETBIRD_MANAGEMENT_ACL_DISCOVERY" -17: echo "RULE=WARP_PURGED_NETBIRD_POLICY_DISCOVERY_ONLY" -24: echo "EDGE_NETBIRD_ACL_DIAG" -31: echo NETBIRD_STATUS -34: echo NETBIRD_JSON_TARGETS -35: netbird status --json >/tmp/771o_netbird_status.json 2>/tmp/771o_netbird_status_json.err || true -39:p=Path(\"/tmp/771o_netbird_status.json\") -42: print(Path(\"/tmp/771o_netbird_status_json.err\").read_text(errors=\"ignore\")[:500] if Path(\"/tmp/771o_netbird_status_json.err\").exists() else \"\") -46:names=(\"e3qxxx\",\"183-106\",\"moldova\",\"usa\") -82: echo CROWDSEC_SAFE_STATE -83: sudo docker ps --format \"{{.Names}} {{.Status}} {{.Ports}}\" | grep -E \"^(crowdsec|socket-proxy|npmplus)\" || true -87: echo "DISCOVER_NETBIRD_MANAGEMENT_ON_KNOWN_HOSTS" -101: echo DOCKER_NETBIRD_CONTAINERS -102: docker ps -a --format "{{.Names}} {{.Image}} {{.Status}} {{.Ports}}" 2>/dev/null | grep -Ei "netbird|management|signal|relay|coturn|zitadel|dashboard" || true -103: echo SYSTEMD_NETBIRD -104: systemctl list-units --all --no-pager 2>/dev/null | grep -Ei "netbird|signal|relay|coturn|zitadel" || true -106: find /opt /etc /var/lib -maxdepth 5 \( -iname "*netbird*" -o -iname "*management*.json" -o -iname "store.json" \) 2>/dev/null | sed -n "1,120p" -111: echo DOCKER_NETBIRD_CONTAINERS -112: docker ps -a --format \"{{.Names}} {{.Image}} {{.Status}} {{.Ports}}\" 2>/dev/null | grep -Ei \"netbird|management|signal|relay|coturn|zitadel|dashboard\" || true -113: echo SYSTEMD_NETBIRD -114: systemctl list-units --all --no-pager 2>/dev/null | grep -Ei \"netbird|signal|relay|coturn|zitadel\" || true -116: find /opt /etc /var/lib -maxdepth 5 \( -iname \"*netbird*\" -o -iname \"*management*.json\" -o -iname \"store.json\" \) 2>/dev/null | sed -n \"1,120p\" -122: echo "TRY_PARSE_NETBIRD_STORE_IF_FOUND_LOCALLY" -124: candidates=$(find /opt /etc /var/lib -maxdepth 7 \( -path "*netbird*" -a \( -iname "store.json" -o -iname "*.db" -o -iname "*.json" \) \) 2>/dev/null | sed -n "1,40p") -163: echo "STATUS=OK_771O_DISCOVERY_DONE_NEXT_FIX_NETBIRD_POLICY_OR_VPS_AGENT" -DEPENDENCY_FILE|path=/root/771p_netbird_vps_peer_repair.sh|mtime=2026-07-02 19:32:10.024856543 +0300|sha256=f37aac25cac5b1c3983392070ddaff09e28497efa28884b03d23efc09a825747 -8:PROOF="/root/evidence/771P_NETBIRD_VPS_PEER_REPAIR_${TS}_PROOF.txt" -11:MOLDOVA_NB=[IP] -16: echo "STEP=771P_NETBIRD_VPS_PEER_REPAIR" -19: echo "RULE=NO_WARP_FIX_NETBIRD_USA_MOLDOVA_PEERS_FOR_CROWDSEC_CAPI" -23: echo "CONFIG_CHANGE=YES_RESTART_NETBIRD_ON_VPS_IF_PUBLIC_SSH_FOUND" -26: echo "LOCAL_PVE01_NETBIRD_STATUS" -29: for ip in "$USA_NB" "$MOLDOVA_NB"; do -35: echo "EDGE_NETBIRD_STATUS_AND_TARGETS" -42: echo NETBIRD_STATUS -46: netbird status --json >/tmp/771p_edge_nb.json 2>/tmp/771p_edge_nb.err || true -57: ip=peer.get(\"netbirdIp\",\"\") -59: print(\"PEER\", fqdn, ip, \"status=\"+str(peer.get(\"status\")), \"handshake=\"+str(peer.get(\"lastWireguardHandshake\")), \"relay=\"+str(peer.get(\"relayAddress\")), \"conn=\"+str(peer.get(\"connectionType\"))) -80: grep -nEi 'e3qxxx|183-106|moldova|молдов|usa|america|vps|netbird|alexhost|aeza|hetzner|public ip|external ip|ssh' /etc/pve/31_HOMELAB_REFERENCE.md 2>/dev/null | sed -n '1,260p' || true -81: echo "SOURCE=systemd_netbird_vps" -84: grep -RHEi 'Host |HostName|User |Port |e3qxxx|183-106|moldova|usa|vps' [SENSITIVE_PATH] /etc/ssh/ssh_config /etc/ssh/ssh_config.d/* 2>/dev/null | sed -n '1,260p' || true -110: if any(x in low for x in ["183-106", "moldova", "молдов", "chisinau"]): -111: profile = "moldova" -123: if "netbird.selfhosted" in host: -138: echo "TEST_PUBLIC_SSH_AND_RESTART_NETBIRD_ON_VPS_IF_FOUND" -159: systemctl restart netbird 2>/dev/null || true -173: echo "CROWDSEC_SAFE_POSTCHECK" -175: sudo docker ps --format \"{{.Names}} {{.Status}} {{.Ports}}\" | grep -E \"^(crowdsec|socket-proxy|npmplus)\" || true -182: echo "STATUS=OK_771P_VPS_PUBLIC_ACCESS_FOUND_RESTARTED_NETBIRD_REVIEW_RECHECK" -DEPENDENCY_FILE|path=/root/771q_netbird_public_vps_repair_then_capi.sh|mtime=2026-07-02 19:45:41.256069566 +0300|sha256=0b6e42e5b4013f7ee2191b140c5c507877fb05e2a4fdd7b39bea7b7c6b108071 -8:PROOF="/root/evidence/771Q_NETBIRD_PUBLIC_VPS_REPAIR_THEN_CAPI_${TS}_PROOF.txt" -11:MOLDOVA_NB=[IP] -16: echo "STEP=771Q_NETBIRD_PUBLIC_VPS_REPAIR_THEN_CAPI" -19: echo "RULE=NO_WARP_REPAIR_NETBIRD_USA_MOLDOVA_THEN_CROWDSEC_CAPI" -23: echo "CONFIG_CHANGE=YES_RESTART_PUBLIC_VPS_NETBIRD_AND_CAPI_IF_ROUTE_READY" -38: "echo SSH_OK; echo HOSTNAME=\$(hostname); command -v netbird >/dev/null 2>&1 && netbird status 2>&1 || echo NETBIRD_CLI_MISSING; curl -4 -sk --connect-timeout 8 --max-time 20 -o /tmp/771q_capi.body -w 'DIRECT_CAPI_HTTP=%{http_code}\n' https://api.crowdsec.net/v3/watchers/login || true" \ -58: "moldova|debian|netbird.vpn.gram1.ru" \ -59: "moldova|root|netbird.vpn.gram1.ru" \ -60: "moldova|debian|[IP]" \ -61: "moldova|root|[IP]" \ -62: "netbird-ui|debian|nb.pvepro.ru" \ -63: "netbird-ui|root|nb.pvepro.ru" -74: echo "RESTART_NETBIRD_ON_REAL_PUBLIC_VPS_ONLY" -79: netbird.vpn.gram1.ru|[IP]|nb.pvepro.ru) -87: "echo BEFORE_HOSTNAME=\$(hostname); command -v netbird >/dev/null 2>&1 && netbird status 2>&1 || true; (sudo systemctl restart netbird 2>/dev/null || systemctl restart netbird 2>/dev/null || true); sleep 15; echo AFTER; command -v netbird >/dev/null 2>&1 && netbird status 2>&1 || true; curl -4 -sk --connect-timeout 8 --max-time 20 -o /tmp/771q_capi_after.body -w 'DIRECT_CAPI_HTTP_AFTER=%{http_code}\n' https://api.crowdsec.net/v3/watchers/login || true" \ -106: echo "RECHECK_EDGE_USA_MOLDOVA_AFTER_PUBLIC_RESTART" -109: systemctl restart netbird 2>/dev/null || true -112: netbird status --json >/tmp/771q_edge_nb.json 2>/tmp/771q_edge_nb.err || true -121: ip=peer.get(\"netbirdIp\",\"\") -123: print(\"PEER\", fqdn, ip, \"status=\"+str(peer.get(\"status\")), \"handshake=\"+str(peer.get(\"lastWireguardHandshake\")), \"relay=\"+str(peer.get(\"relayAddress\")), \"conn=\"+str(peer.get(\"connectionType\"))) -139: echo "EDGE_BUILD_NETBIRD_EGRESS_AND_REGISTER_CAPI_IF_REACHABLE" -144:STACK=/opt/stacks/crowdsec -146:EGRESS_DIR=/etc/homelab-crowdsec-capi-netbird-egress -147:HEALTH_DIR=/var/lib/homelab-health -148:HEALTH_FILE="$HEALTH_DIR/crowdsec-capi.txt" -151:mkdir -p "$EGRESS_DIR/profiles" "$HEALTH_DIR" -158: docker network inspect crowdsec_default --format '{{(index .IPAM.Config 0).Gateway}}' 2>/dev/null || echo "[IP]" -161:write_health() { -167: echo "CHECK=crowdsec-capi" -170: } > "$HEALTH_FILE" -177: crowdsec: -198: docker compose up -d crowdsec -206: health="$(curl -sk --connect-timeout 3 --max-time 6 -o /tmp/771q_health.html -w '%{http_code}' http://[IP]:8088/health || true)" -209: echo "LAPI_TRY=$n HEALTH=$health LAPI_RC=${rc:-NA}" -211: test "$health" = "200" && test "${rc:-NA}" = "0" && return 0 -241:remove("/opt/stacks/crowdsec/config/config.yaml") -DEPENDENCY_FILE|path=/root/771r_edge_netbird_egress_capi_20260702T165517Z.sh|mtime=2026-07-02 19:55:17.909628261 +0300|sha256=9c6f911768869c984ec41016b3134be75620100fac8c60a8b21ef6bde58c6868 -5:STACK=/opt/stacks/crowdsec -7:EGRESS_DIR=/etc/homelab-crowdsec-capi-netbird-egress -8:HEALTH_DIR=/var/lib/homelab-health -9:HEALTH_FILE="$HEALTH_DIR/crowdsec-capi.txt" -12:mkdir -p "$EGRESS_DIR/profiles" "$HEALTH_DIR" -19: docker network inspect crowdsec_default --format '{{(index .IPAM.Config 0).Gateway}}' 2>/dev/null || echo "[IP]" -22:write_health() { -29: echo "CHECK=crowdsec-capi" -32: } > "$HEALTH_FILE" -39: crowdsec: -62: docker compose up -d crowdsec -66: local label n health out rc fatal -71: health="$(curl -sk --connect-timeout 3 --max-time 6 -o /tmp/771r_health.html -w '%{http_code}' http://[IP]:8088/health || true)" -74: fatal="$(docker logs --since 30s crowdsec 2>&1 | grep -Eic 'fatal|unable to run local API|failed to get docker info|lookup socket-proxy|server misbehaving|mapping value is not allowed' || true)" -75: echo "LAPI_TRY=$n HEALTH=$health LAPI_RC=${rc:-NA} FATAL_COUNT_30S=$fatal" -77: if test "$health" = "200" && test "${rc:-NA}" = "0" && test "$fatal" = "0"; then -114:remove("/opt/stacks/crowdsec/config/config.yaml") -115:remove("/opt/stacks/crowdsec/config/user.yaml") -122:p = Path("/opt/stacks/crowdsec/config/config.yaml") -153: " credentials_path: /etc/crowdsec/online_api_credentials.yaml", -166: crowdsec: -171: NO_PROXY: "[IP],localhost,[IP],[IPV6],socket-proxy,crowdsec" -196:test_target_ssh_egress() { -202: echo "TEST_TARGET_SSH_EGRESS profile=$profile ip=$ip" -212: "echo SSH_OK; hostname; curl -4 -sk --connect-timeout 8 --max-time 20 -o /tmp/771r_capi.body -w 'DIRECT_CAPI_HTTP=%{http_code}\n' https://api.crowdsec.net/v3/watchers/login || true" \ -234: code="$(curl -sk --proxy "socks5h://[IP]:${port}" --connect-timeout 12 --max-time 45 -o /tmp/771r_capi_proxy.body -w '%{http_code}' https://api.crowdsec.net/v3/watchers/login || true)" -255:install_egress_service() { -261: echo "INSTALL_EGRESS profile=$profile user=$user ip=$ip bridge=$b" -263: mkdir -p "$EGRESS_DIR/profiles" -264: cat >"$EGRESS_DIR/profiles/${profile}.env" </etc/systemd/system/homelab-crowdsec-capi-netbird-egress.service <<'EOF' -275:Description=Homelab CrowdSec CAPI NetBird VPS SOCKS egress -276:After=network-online.target docker.service netbird.service -282:EnvironmentFile=/etc/homelab-crowdsec-capi-netbird-egress/current.env -291: cat >/usr/local/sbin/homelab-crowdsec-capi-egress-switch <<'EOF' -DEPENDENCY_FILE|path=/root/771r_netbird_public_vps_fixed_then_capi.sh|mtime=2026-07-02 19:55:17.904566554 +0300|sha256=dc8119b6815d60e08f442e77faba6ce6e9899c4120d78490886e222b8510efa7 -8:PROOF="/root/evidence/771R_NETBIRD_PUBLIC_VPS_FIXED_THEN_CAPI_${TS}_PROOF.txt" -10:EDGE_REMOTE="/tmp/771r_edge_netbird_egress_capi_${TS}.sh" -11:EDGE_LOCAL="/root/771r_edge_netbird_egress_capi_${TS}.sh" -31: "echo SSH_OK; echo USER=\$(id -un); echo HOSTNAME=\$(hostname); command -v netbird >/dev/null 2>&1 && netbird status 2>&1 || echo NETBIRD_CLI_MISSING; curl -4 -sk --connect-timeout 8 --max-time 20 -o /tmp/771r_capi.body -w 'DIRECT_CAPI_HTTP=%{http_code}\n' https://api.crowdsec.net/v3/watchers/login || true" \ -49:STACK=/opt/stacks/crowdsec -51:EGRESS_DIR=/etc/homelab-crowdsec-capi-netbird-egress -52:HEALTH_DIR=/var/lib/homelab-health -53:HEALTH_FILE="$HEALTH_DIR/crowdsec-capi.txt" -56:mkdir -p "$EGRESS_DIR/profiles" "$HEALTH_DIR" -63: docker network inspect crowdsec_default --format '{{(index .IPAM.Config 0).Gateway}}' 2>/dev/null || echo "[IP]" -66:write_health() { -73: echo "CHECK=crowdsec-capi" -76: } > "$HEALTH_FILE" -83: crowdsec: -106: docker compose up -d crowdsec -110: local label n health out rc fatal -115: health="$(curl -sk --connect-timeout 3 --max-time 6 -o /tmp/771r_health.html -w '%{http_code}' http://[IP]:8088/health || true)" -118: fatal="$(docker logs --since 30s crowdsec 2>&1 | grep -Eic 'fatal|unable to run local API|failed to get docker info|lookup socket-proxy|server misbehaving|mapping value is not allowed' || true)" -119: echo "LAPI_TRY=$n HEALTH=$health LAPI_RC=${rc:-NA} FATAL_COUNT_30S=$fatal" -121: if test "$health" = "200" && test "${rc:-NA}" = "0" && test "$fatal" = "0"; then -158:remove("/opt/stacks/crowdsec/config/config.yaml") -159:remove("/opt/stacks/crowdsec/config/user.yaml") -166:p = Path("/opt/stacks/crowdsec/config/config.yaml") -197: " credentials_path: /etc/crowdsec/online_api_credentials.yaml", -210: crowdsec: -215: NO_PROXY: "[IP],localhost,[IP],[IPV6],socket-proxy,crowdsec" -240:test_target_ssh_egress() { -246: echo "TEST_TARGET_SSH_EGRESS profile=$profile ip=$ip" -256: "echo SSH_OK; hostname; curl -4 -sk --connect-timeout 8 --max-time 20 -o /tmp/771r_capi.body -w 'DIRECT_CAPI_HTTP=%{http_code}\n' https://api.crowdsec.net/v3/watchers/login || true" \ -278: code="$(curl -sk --proxy "socks5h://[IP]:${port}" --connect-timeout 12 --max-time 45 -o /tmp/771r_capi_proxy.body -w '%{http_code}' https://api.crowdsec.net/v3/watchers/login || true)" -299:install_egress_service() { -305: echo "INSTALL_EGRESS profile=$profile user=$user ip=$ip bridge=$b" -307: mkdir -p "$EGRESS_DIR/profiles" -308: cat >"$EGRESS_DIR/profiles/${profile}.env" </etc/systemd/system/homelab-crowdsec-capi-netbird-egress.service <<'EOF' -DEPENDENCY_FILE|path=/root/771s_edge_capi_netbird_20260702T170251Z.sh|mtime=2026-07-02 20:02:51.044349876 +0300|sha256=c4783c14a3a132616af5db6a065270ccd39a7690f9fd38d0ef27fe5de6bd07e5 -6:PROFILE="${3:-moldova}" -7:STACK=/opt/stacks/crowdsec -9:EGRESS_DIR=/etc/homelab-crowdsec-capi-netbird-egress -10:HEALTH_DIR=/var/lib/homelab-health -11:HEALTH_FILE="$HEALTH_DIR/crowdsec-capi.txt" -14:mkdir -p "$EGRESS_DIR/profiles" "$HEALTH_DIR" -21: docker network inspect crowdsec_default --format '{{(index .IPAM.Config 0).Gateway}}' 2>/dev/null || echo "[IP]" -24:write_health() { -30: echo "CHECK=crowdsec-capi" -33: } > "$HEALTH_FILE" -40: crowdsec: -62: docker compose up -d crowdsec -70: health="$(curl -sk --connect-timeout 3 --max-time 6 -o /tmp/771s_health.html -w '%{http_code}' http://[IP]:8088/health || true)" -73: fatal="$(docker logs --since 30s crowdsec 2>&1 | grep -Eic 'fatal|unable to run local API|failed to get docker info|lookup socket-proxy|server misbehaving|mapping value is not allowed' || true)" -74: echo "LAPI_TRY=$n HEALTH=$health LAPI_RC=${rc:-NA} FATAL_COUNT_30S=$fatal" -76: if test "$health" = "200" && test "${rc:-NA}" = "0" && test "$fatal" = "0"; then -113:remove("/opt/stacks/crowdsec/config/config.yaml") -114:remove("/opt/stacks/crowdsec/config/user.yaml") -121:p=Path("/opt/stacks/crowdsec/config/config.yaml") -153: " credentials_path: /etc/crowdsec/online_api_credentials.yaml", -165: crowdsec: -170: NO_PROXY: "[IP],localhost,[IP],[IPV6],socket-proxy,crowdsec" -195:test_target_ssh_egress() { -199: echo "TEST_TARGET_SSH_EGRESS profile=$profile ip=$ip" -209: "echo SSH_OK; hostname; curl -4 -sk --connect-timeout 8 --max-time 20 -o /tmp/771s_capi.body -w 'DIRECT_CAPI_HTTP=%{http_code}\n' https://api.crowdsec.net/v3/watchers/login || true" \ -231: code="$(curl -sk --proxy "socks5h://[IP]:${port}" --connect-timeout 12 --max-time 45 -o /tmp/771s_capi_proxy.body -w '%{http_code}' https://api.crowdsec.net/v3/watchers/login || true)" -252:install_egress_service() { -257: echo "INSTALL_EGRESS profile=$profile user=$user ip=$ip bridge=$b" -259: mkdir -p "$EGRESS_DIR/profiles" -260: cat >"$EGRESS_DIR/profiles/${profile}.env" </etc/systemd/system/homelab-crowdsec-capi-netbird-egress.service <<'EOF' -271:Description=Homelab CrowdSec CAPI NetBird VPS SOCKS egress -272:After=network-online.target docker.service netbird.service -278:EnvironmentFile=/etc/homelab-crowdsec-capi-netbird-egress/current.env -287: cat >/usr/local/sbin/homelab-crowdsec-capi-egress-switch <<'EOF' -DEPENDENCY_FILE|path=/root/771s_vps_identity_repair_netbird_capi.sh|mtime=2026-07-02 20:02:51.038348586 +0300|sha256=16121a810320b1517291830ba128baf6c68e7b3e6f78786481dedb78a1f061b9 -8:PROOF="/root/evidence/771S_VPS_IDENTITY_REPAIR_NETBIRD_CAPI_${TS}_PROOF.txt" -11:VPS_ALIAS=netbird.vpn.gram1.ru -13:EDGE_REMOTE="/tmp/771s_edge_capi_netbird_${TS}.sh" -14:EDGE_LOCAL="/root/771s_edge_capi_netbird_${TS}.sh" -28:PROFILE="${3:-moldova}" -29:STACK=/opt/stacks/crowdsec -31:EGRESS_DIR=/etc/homelab-crowdsec-capi-netbird-egress -32:HEALTH_DIR=/var/lib/homelab-health -33:HEALTH_FILE="$HEALTH_DIR/crowdsec-capi.txt" -36:mkdir -p "$EGRESS_DIR/profiles" "$HEALTH_DIR" -43: docker network inspect crowdsec_default --format '{{(index .IPAM.Config 0).Gateway}}' 2>/dev/null || echo "[IP]" -46:write_health() { -52: echo "CHECK=crowdsec-capi" -55: } > "$HEALTH_FILE" -62: crowdsec: -84: docker compose up -d crowdsec -92: health="$(curl -sk --connect-timeout 3 --max-time 6 -o /tmp/771s_health.html -w '%{http_code}' http://[IP]:8088/health || true)" -95: fatal="$(docker logs --since 30s crowdsec 2>&1 | grep -Eic 'fatal|unable to run local API|failed to get docker info|lookup socket-proxy|server misbehaving|mapping value is not allowed' || true)" -96: echo "LAPI_TRY=$n HEALTH=$health LAPI_RC=${rc:-NA} FATAL_COUNT_30S=$fatal" -98: if test "$health" = "200" && test "${rc:-NA}" = "0" && test "$fatal" = "0"; then -135:remove("/opt/stacks/crowdsec/config/config.yaml") -136:remove("/opt/stacks/crowdsec/config/user.yaml") -143:p=Path("/opt/stacks/crowdsec/config/config.yaml") -175: " credentials_path: /etc/crowdsec/online_api_credentials.yaml", -187: crowdsec: -192: NO_PROXY: "[IP],localhost,[IP],[IPV6],socket-proxy,crowdsec" -217:test_target_ssh_egress() { -221: echo "TEST_TARGET_SSH_EGRESS profile=$profile ip=$ip" -231: "echo SSH_OK; hostname; curl -4 -sk --connect-timeout 8 --max-time 20 -o /tmp/771s_capi.body -w 'DIRECT_CAPI_HTTP=%{http_code}\n' https://api.crowdsec.net/v3/watchers/login || true" \ -253: code="$(curl -sk --proxy "socks5h://[IP]:${port}" --connect-timeout 12 --max-time 45 -o /tmp/771s_capi_proxy.body -w '%{http_code}' https://api.crowdsec.net/v3/watchers/login || true)" -274:install_egress_service() { -279: echo "INSTALL_EGRESS profile=$profile user=$user ip=$ip bridge=$b" -281: mkdir -p "$EGRESS_DIR/profiles" -282: cat >"$EGRESS_DIR/profiles/${profile}.env" </etc/systemd/system/homelab-crowdsec-capi-netbird-egress.service <<'EOF' -DEPENDENCY_FILE|path=/root/771t_temp_hostkey_netbird_identity_then_capi.sh|mtime=2026-07-02 20:10:05.923400398 +0300|sha256=0c3d628b23b67849d52112322ab2e1ee3779012c10578a0b8cece0d8ddf639da -8:PROOF="/root/evidence/771T_TEMP_HOSTKEY_NETBIRD_IDENTITY_THEN_CAPI_${TS}_PROOF.txt" -29:STACK=/opt/stacks/crowdsec -31:EGRESS_DIR=/etc/homelab-crowdsec-capi-netbird-egress -32:HEALTH_DIR=/var/lib/homelab-health -33:HEALTH_FILE="$HEALTH_DIR/crowdsec-capi.txt" -36:mkdir -p "$EGRESS_DIR/profiles" "$HEALTH_DIR" -43: docker network inspect crowdsec_default --format '{{(index .IPAM.Config 0).Gateway}}' 2>/dev/null || echo "[IP]" -46:write_health() { -50: echo "CHECK=crowdsec-capi" -53: } > "$HEALTH_FILE" -60: crowdsec: -82: docker compose up -d crowdsec -90: health="$(curl -sk --connect-timeout 3 --max-time 6 -o /tmp/771t_health.html -w '%{http_code}' http://[IP]:8088/health || true)" -93: fatal="$(docker logs --since 30s crowdsec 2>&1 | grep -Eic 'fatal|unable to run local API|failed to get docker info|lookup socket-proxy|server misbehaving|mapping value is not allowed' || true)" -94: echo "LAPI_TRY=$n HEALTH=$health LAPI_RC=${rc:-NA} FATAL_COUNT_30S=$fatal" -96: if test "$health" = "200" && test "${rc:-NA}" = "0" && test "$fatal" = "0"; then -131:remove("/opt/stacks/crowdsec/config/config.yaml") -132:remove("/opt/stacks/crowdsec/config/user.yaml") -139:p=Path("/opt/stacks/crowdsec/config/config.yaml") -169: " credentials_path: /etc/crowdsec/online_api_credentials.yaml", -181: crowdsec: -186: NO_PROXY: "[IP],localhost,[IP],[IPV6],socket-proxy,crowdsec" -211:install_egress() { -213: echo "INSTALL_EGRESS target=$TARGET_NB user=$TARGET_USER profile=$PROFILE bridge=$b" -215: echo "VERIFY_NETBIRD_SSH_TO_TARGET" -218: "echo SSH_OK; hostname; curl -4 -sk --connect-timeout 8 --max-time 20 -o /tmp/771t_capi_direct.body -w 'DIRECT_CAPI_HTTP=%{http_code}\n' https://api.crowdsec.net/v3/watchers/login || true" >"$out" 2>&1 -226: mkdir -p "$EGRESS_DIR/profiles" -227: cat >"$EGRESS_DIR/profiles/${PROFILE}.env" </etc/systemd/system/homelab-crowdsec-capi-netbird-egress.service <<'EOF' -238:Description=Homelab CrowdSec CAPI NetBird VPS SOCKS egress -239:After=network-online.target docker.service netbird.service -245:EnvironmentFile=/etc/homelab-crowdsec-capi-netbird-egress/current.env -254: cat >/usr/local/sbin/homelab-crowdsec-capi-egress-switch <<'EOF' -258:base=/etc/homelab-crowdsec-capi-netbird-egress -266:systemctl restart homelab-crowdsec-capi-netbird-egress.service -DEPENDENCY_FILE|path=/root/771u_find_key_fix_netbird_egress_capi.sh|mtime=2026-07-02 20:19:23.540526618 +0300|sha256=6b6511ec3d614793e3542777ccfcb5e8c5f09cff77ee2d57624b3ac980787bcb -8:PROOF="/root/evidence/771U_FIND_KEY_FIX_NETBIRD_EGRESS_CAPI_${TS}_PROOF.txt" -29:STACK=/opt/stacks/crowdsec -31:EGRESS_DIR=/etc/homelab-crowdsec-capi-netbird-egress -32:HEALTH_DIR=/var/lib/homelab-health -33:HEALTH_FILE="$HEALTH_DIR/crowdsec-capi.txt" -34:SSH_KEY="$EGRESS_DIR/id_ed25519" -37:mkdir -p "$EGRESS_DIR/profiles" "$HEALTH_DIR" -38:chmod 700 "$EGRESS_DIR" -45: docker network inspect crowdsec_default --format '{{(index .IPAM.Config 0).Gateway}}' 2>/dev/null || echo "[IP]" -48:write_health() { -52: echo "CHECK=crowdsec-capi" -55: } > "$HEALTH_FILE" -62: crowdsec: -84: docker compose up -d crowdsec -92: health="$(curl -sk --connect-timeout 3 --max-time 6 -o /tmp/771u_health.html -w '%{http_code}' http://[IP]:8088/health || true)" -95: fatal="$(docker logs --since 30s crowdsec 2>&1 | grep -Eic 'fatal|unable to run local API|failed to get docker info|lookup socket-proxy|server misbehaving|mapping value is not allowed' || true)" -96: echo "LAPI_TRY=$n HEALTH=$health LAPI_RC=${rc:-NA} FATAL_COUNT_30S=$fatal" -98: if test "$health" = "200" && test "${rc:-NA}" = "0" && test "$fatal" = "0"; then -133:remove("/opt/stacks/crowdsec/config/config.yaml") -134:remove("/opt/stacks/crowdsec/config/user.yaml") -141:p=Path("/opt/stacks/crowdsec/config/config.yaml") -170: " credentials_path: /etc/crowdsec/online_api_credentials.yaml", -182: crowdsec: -187: NO_PROXY: "[IP],localhost,[IP],[IPV6],socket-proxy,crowdsec" -213: ssh-keygen -q -t ed25519 -N "" -C "homelab-crowdsec-capi-netbird-egress-edge" -f "$SSH_KEY" -217: echo "EDGE_EGRESS_PUBKEY_FINGERPRINT" -221:install_egress() { -223: echo "INSTALL_EGRESS target=$TARGET_NB user=$TARGET_USER profile=$PROFILE bridge=$b" -225: echo "VERIFY_NETBIRD_SSH_TO_TARGET" -234: "echo SSH_OK; hostname; curl -4 -sk --connect-timeout 8 --max-time 20 -o /tmp/771u_capi_direct.body -w 'DIRECT_CAPI_HTTP=%{http_code}\n' https://api.crowdsec.net/v3/watchers/login || true" >"$out" 2>&1 -242: mkdir -p "$EGRESS_DIR/profiles" -243: cat >"$EGRESS_DIR/profiles/${PROFILE}.env" </etc/systemd/system/homelab-crowdsec-capi-netbird-egress.service <<'EOF' -255:Description=Homelab CrowdSec CAPI NetBird VPS SOCKS egress -256:After=network-online.target docker.service netbird.service -DEPENDENCY_FILE|path=/root/771v_deep_key_backup_netbird_capi.sh|mtime=2026-07-02 20:31:17.990220570 +0300|sha256=92191ce6124981ee0468cc8f95c749c47bca5fc61f9aecaeedeef91cdd434170 -8:PROOF="/root/evidence/771V_DEEP_KEY_BACKUP_NETBIRD_CAPI_${TS}_PROOF.txt" -11:EDGE_LOCAL="/root/771v_edge_netbird_capi_${TS}.sh" -12:EDGE_REMOTE="/tmp/771v_edge_netbird_capi_${TS}.sh" -29:STACK=/opt/stacks/crowdsec -31:EGRESS_DIR=/etc/homelab-crowdsec-capi-netbird-egress -32:HEALTH_DIR=/var/lib/homelab-health -33:HEALTH_FILE="$HEALTH_DIR/crowdsec-capi.txt" -34:SSH_KEY="$EGRESS_DIR/id_ed25519" -37:mkdir -p "$EGRESS_DIR/profiles" "$HEALTH_DIR" -38:chmod 700 "$EGRESS_DIR" -45: docker network inspect crowdsec_default --format '{{(index .IPAM.Config 0).Gateway}}' 2>/dev/null || echo "[IP]" -48:write_health() { -52: echo "CHECK=crowdsec-capi" -55: } > "$HEALTH_FILE" -62: crowdsec: -84: docker compose up -d crowdsec -92: health="$(curl -sk --connect-timeout 3 --max-time 6 -o /tmp/771v_health.html -w '%{http_code}' http://[IP]:8088/health || true)" -95: fatal="$(docker logs --since 30s crowdsec 2>&1 | grep -Eic 'fatal|unable to run local API|failed to get docker info|lookup socket-proxy|server misbehaving|mapping value is not allowed' || true)" -96: echo "LAPI_TRY=$n HEALTH=$health LAPI_RC=${rc:-NA} FATAL_COUNT_30S=$fatal" -98: if test "$health" = "200" && test "${rc:-NA}" = "0" && test "$fatal" = "0"; then -133:remove("/opt/stacks/crowdsec/config/config.yaml") -134:remove("/opt/stacks/crowdsec/config/user.yaml") -141:p = Path("/opt/stacks/crowdsec/config/config.yaml") -172: " credentials_path: /etc/crowdsec/online_api_credentials.yaml", -184: crowdsec: -189: NO_PROXY: "[IP],localhost,[IP],[IPV6],socket-proxy,crowdsec" -216: ssh-keygen -q -t ed25519 -N "" -C "homelab-crowdsec-capi-netbird-egress-edge" -f "$SSH_KEY" -220: echo "EDGE_EGRESS_PUBKEY_FINGERPRINT" -224:install_egress() { -226: echo "INSTALL_EGRESS target=$TARGET_NB user=$TARGET_USER profile=$PROFILE bridge=$b" -228: echo "VERIFY_NETBIRD_SSH_TO_TARGET" -237: "echo SSH_OK; hostname; curl -4 -sk --connect-timeout 8 --max-time 20 -o /tmp/771v_capi_direct.body -w 'DIRECT_CAPI_HTTP=%{http_code}\n' https://api.crowdsec.net/v3/watchers/login || true" >"$out" 2>&1 -245: mkdir -p "$EGRESS_DIR/profiles" -246: cat >"$EGRESS_DIR/profiles/${PROFILE}.env" </etc/systemd/system/homelab-crowdsec-capi-netbird-egress.service <<'EOF' -DEPENDENCY_FILE|path=/root/771z_after_manual_netbird_egress_capi.sh|mtime=2026-07-02 21:37:47.943243883 +0300|sha256=f198ed621fd726ed6f15c4a0dd49fad307befd01e7a58ebbda1ddc379f5b49ee -8:PROOF="/root/evidence/771Z_AFTER_MANUAL_NETBIRD_EGRESS_CAPI_${TS}_PROOF.txt" -14: echo "STEP=771Z_AFTER_MANUAL_NETBIRD_EGRESS_CAPI" -17: echo "RULE=VERIFY_RELAY_MAIL_EGRESS_AND_REGISTER_CROWDSEC_CAPI" -18: echo "TARGETS=edge-vm:[IP] relay-moldova:[IP] mail-usa:[IP]" -25:STACK=/opt/stacks/crowdsec -27:HEALTH=/var/lib/homelab-health/crowdsec-capi.txt -28:mkdir -p /var/lib/homelab-health -34:write_health() { -38: echo "CHECK=crowdsec-capi" -41: } > "$HEALTH" -49: health="$(curl -sk --connect-timeout 3 --max-time 6 -o /tmp/771z_health.html -w '%{http_code}' http://[IP]:8088/health || true)" -52: echo "LAPI_TRY=$n HEALTH=$health LAPI_RC=${rc:-NA}" -54: test "$health" = "200" && test "${rc:-NA}" = "0" && return 0 -63: crowdsec: -84: docker compose up -d crowdsec -91: crowdsec: -146:remove("/opt/stacks/crowdsec/config/config.yaml") -147:remove("/opt/stacks/crowdsec/config/user.yaml") -154:p = Path("/opt/stacks/crowdsec/config/config.yaml") -183: " credentials_path: /etc/crowdsec/online_api_credentials.yaml", -194:echo "NETBIRD_RESTART" -195:systemctl restart netbird 2>/dev/null || true -198:echo "NETBIRD_STATUS" -199:netbird status 2>&1 | redact || true -201:echo "PEERS_RELAY_MAIL_EDGE" -202:netbird status --json >/tmp/771z_nb.json 2>/tmp/771z_nb.err || true -211: ip=peer.get("netbirdIp","") -212: if ip in {"[IP]","[IP]","[IP]"} or any(x in fqdn for x in ["relay","mail","edge-vm"]): -216:echo "RELAY_MAIL_CONNECTIVITY" -224:echo "WG_ALLOWED_IPS_RELAY_MAIL" -226: wg show wt0 allowed-ips 2>/dev/null | sed -E 's/^[A-Za-z0-9+\/=]{20,}/PEERKEY_REDACTED/' | grep -E '100\.100\.19\.1|100\.100\.147\.204|0\.0\.0\.0/0|100\.100\.0\.0/16' || echo RELAY_MAIL_OR_DEFAULT_NOT_IN_ALLOWED_IPS -229:echo "CAPI_DIRECT_TEST_AFTER_MANUAL_NETBIRD" -230:capi_code="$(curl -4 -sk --http1.1 --connect-timeout 12 --max-time 45 -o /tmp/771z_capi.body -w '%{http_code}' https://api.crowdsec.net/v3/watchers/login || true)" -240: write_health "REVIEW_MANUAL_NETBIRD_EGRESS_NOT_READY" "After manual NetBird setup, edge still cannot reach api.crowdsec.net; CrowdSec remains LAPI-only no-capi" -241: echo "EDGE_STATUS=REVIEW_MANUAL_NETBIRD_EGRESS_NOT_READY_CAPI_NOT_DONE" -248:mkdir -p "manual-backups/771z-$TS" -DEPENDENCY_FILE|path=/srv/homelab-ops/kb/current/03_ACCESS_AND_SECRETS.md|mtime=2026-08-21 00:06:53.672037607 +0300|sha256=e750284217dc41b5809268a0bbc54ed0948bdd14680fe2f282f307640937496c -DEPENDENCY_FILE|path=/usr/local/sbin/homelab-alerting-health|mtime=2026-07-06 17:21:33.211361096 +0300|sha256=2050bac8c2ca8f1116e6d4fabe476292253bf38c56a49456242606980b4eb797 -4:OUT="/var/lib/homelab-health/alerting.txt" -5:mkdir -p /var/lib/homelab-health -20:KUMA_STATE="$(ssh -n -o BatchMode=yes -o ConnectTimeout=8 "debian@$MON_HOST" "sudo docker inspect -f '{{.State.Status}} {{if .State.Health}}{{.State.Health.Status}}{{end}}' uptime-kuma" 2>/dev/null || true)" -23:GOTIFY_STATE="$(ssh -n -o BatchMode=yes -o ConnectTimeout=8 "debian@$MON_HOST" "sudo docker inspect -f '{{.State.Status}}' gotify" 2>/dev/null || true)" -24:if echo "$GOTIFY_STATE" | grep -q "running"; then ok gotify_container; else bad gotify_container; fi -26:MON_ROW="$(ssh -n -o BatchMode=yes -o ConnectTimeout=8 "debian@$MON_HOST" "sudo sqlite3 '$KUMA_DB' \"SELECT m.id || '|' || m.active || '|' || m.type || '|' || m.url || '|' || m.keyword || '|' || COALESCE(n.name,'') || '|' || COALESCE(n.active,'') FROM monitor m LEFT JOIN monitor_notification mn ON mn.monitor_id=m.id LEFT JOIN notification n ON n.id=mn.notification_id WHERE m.name='Homelab Overall Health' LIMIT 1;\"" 2>/dev/null || true)" -30:if [[ "${NNAME:-}" = "Gotify" && "${NACTIVE:-}" = "1" ]]; then ok gotify_link; else bad gotify_link; fi -33:echo "STATUS=$STATUS TS=$TS TYPE=alerting BAD=$BAD$DETAILS kuma_state=${KUMA_STATE// /_} gotify_state=${GOTIFY_STATE// /_} monitor=${MON_ROW:-missing}" > "$OUT" -DEPENDENCY_FILE|path=/usr/local/sbin/homelab-kuma-monitor-policy|mtime=2026-07-06 17:21:33.211264665 +0300|sha256=87e1f14bde3e5eb2d0512c949e1e32d7fe3214f38797559b6e3e1a9d855f059a -4:OUT="/var/lib/homelab-health/kuma-monitor-policy.txt" -12:PUBLIC_URL="https://backup.gram1.ru/overall.txt" -14:mkdir -p /var/lib/homelab-health "$ROOT" -26:SQL="SELECT id||'|'||name||'|'||type||'|'||url||'|'||COALESCE(keyword,'')||'|'||active FROM monitor WHERE name IN ('Homelab Overall Health','Homelab Overall Public Health','File Browser Backup','File Browser Offhost Backup','File Browser Restore Validation','NPMplus Kuma Config Backup','Paperless Restore') ORDER BY id;" -30:LINKSQL="SELECT m.name||'|'||COALESCE(n.name,'')||'|'||COALESCE(n.active,'') FROM monitor m LEFT JOIN monitor_notification mn ON mn.monitor_id=m.id LEFT JOIN notification n ON n.id=mn.notification_id WHERE m.name IN ('Homelab Overall Health','Homelab Overall Public Health') ORDER BY m.name,n.name;" -34:printf "%s\n" "$ROWS" | grep -Fq "Homelab Overall Health|keyword|$PRIVATE_URL|STATUS=OK|1" && ok private_monitor || bad private_monitor -35:printf "%s\n" "$ROWS" | grep -Fq "Homelab Overall Public Health|keyword|$PUBLIC_URL|STATUS=OK|1" && ok public_monitor || bad public_monitor -36:printf "%s\n" "$LINKS" | grep -Fxq "Homelab Overall Health|Gotify|1" && ok private_gotify || bad private_gotify -37:printf "%s\n" "$LINKS" | grep -Fxq "Homelab Overall Public Health|Gotify|1" && ok public_gotify || bad public_gotify -39:LEGACY_FOUND="$(printf "%s\n" "$ROWS" | awk -F'|' '$2=="File Browser Backup" || $2=="File Browser Offhost Backup" || $2=="File Browser Restore Validation" || $2=="NPMplus Kuma Config Backup" || $2=="Paperless Restore" {c++} END{print c+0}')" -40:LEGACY_ACTIVE="$(printf "%s\n" "$ROWS" | awk -F'|' '($2=="File Browser Backup" || $2=="File Browser Offhost Backup" || $2=="File Browser Restore Validation" || $2=="NPMplus Kuma Config Backup" || $2=="Paperless Restore") && $6=="1" {c++} END{print c+0}')" -43:KUMA_STATE="$(ssh -n -o BatchMode=yes -o ConnectTimeout=8 "$KUMA_HOST" "sudo docker inspect -f '{{.State.Status}} {{if .State.Health}}{{.State.Health.Status}}{{end}}' uptime-kuma" 2>/dev/null || true)" -DEPENDENCY_FILE|path=/usr/local/sbin/homelab-runbook-generate|mtime=2026-07-03 14:21:01.771069001 +0300|sha256=956056878b1636369f39f8d79a1c725ebde17ee319bbbb5d5b09f188d656b23e -7:health = Path("/var/lib/homelab-health/runbooks.txt") -8:state = Path("/var/lib/homelab-backup/apps") -12:health.parent.mkdir(parents=True, exist_ok=True) -32: f"## Current verified backup state\n" -39: f"- local_staging_deleted_after_two_cloud_verify: {d.get('local_backup_deleted', 'na')}\n\n" -43: f"jq . /var/lib/homelab-backup/apps/latest-{app}.json\n" -44: f"appbackupctl restore-check {app}\n" -50: f"homelab-backupctl audit\n" -53: f"homelab-overall-health\n" -73:index += "homelab-overall-health\n" -74:index += "homelab-backupctl audit\n" -85: "backupctl restore-check\n" -86: "appbackupctl restore-check safety-net-control-plane\n" -91: "cat /var/lib/homelab-health/overall.txt\n" -92: "cat /var/lib/homelab-health/alerting.txt\n" -93: "ssh -n debian@[IP] \"sudo docker ps --format '{{.Names}}|{{.Status}}' | egrep 'uptime-kuma|gotify'\"\n" -96: "If backup verification, dependency map, or alerting health is not OK, stop automation and escalate manually.\n" -108:health.write_text(line + "\n") -DEPENDENCY_FILE|path=/usr/local/sbin/homelab-vps-identity-audit|mtime=2026-06-24 16:00:07.135818676 +0300|sha256=b709691c52ae48432b70e197eb4810b6c374a05911cee3b9fdba4ecc67c76c3a -5:VPS_ALIAS="${VPS_ALIAS:-netbird.vpn.gram1.ru}" -6:H="/var/lib/homelab-health/vps-identity-audit.txt" -9:mkdir -p /var/lib/homelab-health -23:backup="$(find /mnt/staging/netbird-vps-backups/snapshots -maxdepth 2 -type f -name 'netbird-vps-backup.tgz' 2>/dev/null | sort | tail -n1 || true)" -24:backup_hostkeys=NO -25:backup_match=NO -27:if [ -n "$backup" ]; then -28: if tar -tzf "$backup" | grep -Eq 'ssh_host_.*_key\.pub$'; then -29: backup_hostkeys=YES -31: tar -xOzf "$backup" "$k" >> "$TMP/backup-hostkeys.pub" 2>/dev/null || true -32: done < <(tar -tzf "$backup" | grep -E 'ssh_host_.*_key\.pub$') -33: ssh-keygen -lf "$TMP/backup-hostkeys.pub" > "$TMP/backup-fingerprints.txt" 2>/dev/null || true -34: if grep -Fxf "$TMP/current-fingerprints.txt" "$TMP/backup-fingerprints.txt" >/dev/null 2>&1; then -35: backup_match=YES -41:[ "$backup_hostkeys" = "YES" ] && trust="BLOCKED_HOSTKEY_MISMATCH" -42:[ "$backup_match" = "YES" ] && trust="VERIFIED_BACKUP_MATCH" -46: "$status" "$TS" "$VPS_IP" "$VPS_ALIAS" "${alias_ip:-NONE}" "$trust" "$backup_hostkeys" > "$H" -DEPENDENCY_FILE|path=/var/lib/homelab-backup-matrix/compat/netbird_vps_backup--netbird-vps-trust.txt--44dfdc0d5d0a.txt|mtime=2026-07-18 01:52:29.872264457 +0300|sha256=c1f8ec2f8be329eba2bed981a6c281a2b5d932ae36e8440b07895ff7db93084a -3:TYPE=backup-matrix-generated -4:JOB_ID=netbird_vps_backup -5:JOB_NAME=netbird-vps-backup -7:TIMER=netbird-vps-backup.timer -8:SERVICE=netbird-vps-backup.service -11:SOURCE_KIND=health_file -12:EVIDENCE_LEVEL=artifact_or_health -15:SOURCE_HEALTH=/var/lib/homelab-backup-matrix/evidence/netbird_vps_backup/0-netbird-vps-trust.txt -21:LEGACY_SOURCE_PATH=/var/lib/homelab-health/netbird-vps-trust.txt -22:CANONICAL_JOB_ID=netbird_vps_backup -23:CANONICAL_STATE=/var/lib/homelab-backup-matrix/state.json -DEPENDENCY_FILE|path=/var/lib/homelab-backup-matrix/evidence/homelab_verified_backup_health/0-backup-registry-latest.txt|mtime=2026-07-24 06:46:41.644124478 +0300|sha256=606f956d0c95435b4547a71a7ed3c5d8d0461700437815d064e3a7363e19ef21 -22:app=gotify job=homelab-pve01-app-gotify-20260724T014943Z status=success size=64413 deleted=true proxy_hosts=na certs=na -25:app=healthchecks job=homelab-pve01-app-healthchecks-20260724T025849Z status=success size=686 deleted=true proxy_hosts=na certs=na -53:Sat 2026-07-25 [IPV6] MSK 20h Fri 2026-07-24 [IPV6] MSK 3h 33min ago homelab-emergency-backup.timer homelab-emergency-backup.service -54:Sat 2026-07-25 [IPV6] MSK 21h Fri 2026-07-24 [IPV6] MSK 2h 58min ago homelab-appbackup-dockge-stacks.timer homelab-appbackup-dockge-stacks.service -55:Sat 2026-07-25 [IPV6] MSK 21h Fri 2026-07-24 [IPV6] MSK 2h 29min ago homelab-appbackup-npmplus-edge.timer homelab-appbackup-npmplus-edge.service -56:Sat 2026-07-25 [IPV6] MSK 21h Fri 2026-07-24 [IPV6] MSK 2h 0min ago homelab-appbackup-sqlite-batch.timer homelab-appbackup-sqlite-batch.service -57:Sat 2026-07-25 [IPV6] MSK 22h Fri 2026-07-24 [IPV6] MSK 1h 28min ago homelab-appbackup-postgres-batch.timer homelab-appbackup-postgres-batch.service -58:Sat 2026-07-25 [IPV6] MSK 22h Fri 2026-07-24 [IPV6] MSK 52min ago homelab-appbackup-mariadb-batch.timer homelab-appbackup-mariadb-batch.service -63:/var/lib/homelab-backup/quarantine/20260703T093243Z -DEPENDENCY_FILE|path=/var/lib/homelab-backup-matrix/evidence-map.json|mtime=2026-07-10 12:47:24.326276710 +0300|sha256=0735c335850b22ee4947e0940c6cfd1cc1525f340f102b7f5bcfef5069f165c4 -6: "job": "filebrowser-backup", -7: "source_path": "/var/lib/homelab-health/filebrowser-backup.txt", -8: "evidence_path": "/var/lib/homelab-backup-matrix/evidence/filebrowser_backup/0-filebrowser-backup.txt" -12: "source_path": "/var/lib/homelab-health/filebrowser-restore.txt", -13: "evidence_path": "/var/lib/homelab-backup-matrix/evidence/filebrowser_restore_validate/0-filebrowser-restore.txt" -16: "job": "homelab-appbackup-dockge-stacks", -17: "source_path": "/var/lib/homelab-backup/apps/latest-dockge-stacks.json", -18: "evidence_path": "/var/lib/homelab-backup-matrix/evidence/homelab_appbackup_dockge_stacks/0-latest-dockge-stacks.json" -21: "job": "homelab-appbackup-npmplus-edge", -22: "source_path": "/var/lib/homelab-backup/apps/latest-npmplus-edge.json", -23: "evidence_path": "/var/lib/homelab-backup-matrix/evidence/homelab_appbackup_npmplus_edge/0-latest-npmplus-edge.json" -26: "job": "homelab-authentik-backup", -27: "source_path": "/var/lib/homelab-health/authentik-backup.txt", -28: "evidence_path": "/var/lib/homelab-backup-matrix/evidence/homelab_authentik_backup/0-authentik-backup.txt" -31: "job": "homelab-authentik-backup", -32: "source_path": "/var/lib/homelab-health/authentik-offhost.txt", -33: "evidence_path": "/var/lib/homelab-backup-matrix/evidence/homelab_authentik_backup/1-authentik-offhost.txt" -36: "job": "homelab-authentik-backup", -37: "source_path": "/var/lib/homelab-health/authentik-restore.txt", -38: "evidence_path": "/var/lib/homelab-backup-matrix/evidence/homelab_authentik_backup/2-authentik-restore.txt" -41: "job": "homelab-backup-coverage-matrix", -42: "source_path": "/var/lib/homelab-health/backup-coverage-matrix.txt", -43: "evidence_path": "/var/lib/homelab-backup-matrix/evidence/homelab_backup_coverage_matrix/0-backup-coverage-matrix.txt" -46: "job": "homelab-backup-sla-health", -47: "source_path": "/var/lib/homelab-health/backup-sla.txt", -48: "evidence_path": "/var/lib/homelab-backup-matrix/evidence/homelab_backup_sla_health/0-backup-sla.txt" -51: "job": "homelab-edge-vm-vzdump-backup", -52: "source_path": "/var/lib/homelab-health/edge-vm-vzdump-backup.txt", -53: "evidence_path": "/var/lib/homelab-backup-matrix/evidence/homelab_edge_vm_vzdump_backup/0-edge-vm-vzdump-backup.txt" -56: "job": "homelab-emergency-backup", -57: "source_path": "/var/lib/homelab-backup/latest-emergency-pack.json", -58: "evidence_path": "/var/lib/homelab-backup-matrix/evidence/homelab_emergency_backup/0-latest-emergency-pack.json" -61: "job": "homelab-extended-appbackup", -62: "source_path": "/var/lib/homelab-health/extended-appbackup.txt", -63: "evidence_path": "/var/lib/homelab-backup-matrix/evidence/homelab_extended_appbackup/0-extended-appbackup.txt" -66: "job": "homelab-gitea-backup", -DEPENDENCY_FILE|path=/var/lib/homelab-backup-matrix/evidence/netbird_vps_backup/0-netbird-vps-backup.txt|mtime=2026-07-01 04:31:05.433646905 +0300|sha256=372804695d6dba94e29e254f2175701e976692e7b15ee83fdcb6f10e05ca624e -2:SERVICE=netbird-vps-backup -5:SNAPSHOT=/mnt/staging/netbird-vps-backups/snapshots/20260701T013054Z -6:LATEST_SYMLINK=/mnt/staging/netbird-vps-backups/snapshots/latest -DEPENDENCY_FILE|path=/var/lib/homelab-backup-matrix/migration-plan.json|mtime=2026-07-10 13:10:05.080842534 +0300|sha256=260b6ef9d8c3370b6b429694ff642319ece1592f9e7c261e37b9fc3de0d33581 -3: "dpkg-db-backup", -4: "filebrowser-backup", -6: "homelab-appbackup-dockge-stacks", -7: "homelab-appbackup-mariadb-batch", -8: "homelab-appbackup-npmplus-edge", -9: "homelab-appbackup-postgres-batch", -10: "homelab-appbackup-sqlite-batch", -11: "homelab-auto-backup", -12: "homelab-backup-coverage-matrix", -13: "homelab-backup-freshness", -14: "homelab-backup-healthcheck", -15: "homelab-backup-sla-health", -16: "homelab-edge-vm-vzdump-backup", -17: "homelab-emergency-backup", -18: "homelab-extended-appbackup", -19: "homelab-mail-cloud-restore-drill", -20: "homelab-verified-backup-health", -21: "homelab-vm-backup-health", -22: "homelab-vm-backup-policy", -23: "homelab-vm-full-backup", -27: "immich-media-backup", -28: "memos-backup", -29: "netbird-vps-backup", -30: "paperless-backup", -32: "skladchik-reports-monitor-backup-restore-check", -33: "skladchik-reports-monitor-backup" -38: "homelab-appbackup-postgres-batch" -40: "owner_job": "homelab-appbackup-postgres-batch", -41: "path": "/var/lib/homelab-health/authentik-backup.txt" -45: "homelab-appbackup-postgres-batch" -47: "owner_job": "homelab-appbackup-postgres-batch", -48: "path": "/var/lib/homelab-health/authentik-offhost.txt" -52: "homelab-appbackup-postgres-batch" -54: "owner_job": "homelab-appbackup-postgres-batch", -55: "path": "/var/lib/homelab-health/authentik-restore.txt" -59: "homelab-backup-coverage-matrix" -DEPENDENCY_FILE|path=/var/lib/homelab-backup-matrix/rendered/netbird-vps-backup.txt|mtime=2026-08-30 21:08:09.037130131 +0300|sha256=d2d09878e7ccd4159c2c66225fea6bbcf30d7ba9a7e70ab30f5df4fae631620c -3:TYPE=backup-matrix-generated -4:JOB_ID=netbird_vps_backup -5:JOB_NAME=netbird-vps-backup -7:TIMER=netbird-vps-backup.timer -8:SERVICE=netbird-vps-backup.service -11:SOURCE_KIND=health_file -12:EVIDENCE_LEVEL=artifact_or_health -15:SOURCE_HEALTH=/var/lib/homelab-backup-matrix/evidence/netbird_vps_backup/0-netbird-vps-backup.txt -DEPENDENCY_FILE|path=/var/lib/homelab-backup-matrix/state.json|mtime=2026-08-30 21:08:09.112843466 +0300|sha256=bba7caf3207a046105369da4d140f3342dec2e0897d91d5b4e2e099f5aab6b90 -3: "compatibility_map": "/var/lib/homelab-backup-matrix/compat-map.json", -12: "generated_health_output": "/var/lib/homelab-backup-matrix/rendered/dpkg-db-backup.txt", -13: "id": "dpkg_db_backup", -15: "name": "dpkg-db-backup", -16: "service": "dpkg-db-backup.service", -19: "source_health": "systemd:dpkg-db-backup.service", -24: "timer": "dpkg-db-backup.timer", -32: "evidence_level": "artifact_or_health", -34: "generated_health_output": "/var/lib/homelab-backup-matrix/rendered/filebrowser-backup.txt", -35: "id": "filebrowser_backup", -37: "name": "filebrowser-backup", -38: "service": "filebrowser-backup.service", -41: "source_health": "/var/lib/homelab-backup/apps/latest-filebrowser.json", -42: "source_kind": "health_file", -46: "timer": "filebrowser-backup.timer", -54: "evidence_level": "artifact_or_health", -56: "generated_health_output": "/var/lib/homelab-backup-matrix/rendered/filebrowser-restore-validate.txt", -63: "source_health": "/var/lib/homelab-backup-matrix/evidence/filebrowser_restore_validate/0-filebrowser-restore.txt", -64: "source_kind": "health_file", -76: "evidence_level": "artifact_or_health", -78: "generated_health_output": "/var/lib/homelab-backup-matrix/rendered/homelab-appbackup-dockge-stacks.txt", -79: "id": "homelab_appbackup_dockge_stacks", -81: "name": "homelab-appbackup-dockge-stacks", -82: "service": "homelab-appbackup-dockge-stacks.service", -85: "source_health": "/var/lib/homelab-backup-matrix/evidence/homelab_appbackup_dockge_stacks/0-latest-dockge-stacks.json", -86: "source_kind": "health_file", -90: "timer": "homelab-appbackup-dockge-stacks.timer", -100: "generated_health_output": "/var/lib/homelab-backup-matrix/rendered/homelab-appbackup-mariadb-batch.txt", -101: "id": "homelab_appbackup_mariadb_batch", -103: "name": "homelab-appbackup-mariadb-batch", -104: "service": "homelab-appbackup-mariadb-batch.service", -107: "source_health": "systemd:homelab-appbackup-mariadb-batch.service", -112: "timer": "homelab-appbackup-mariadb-batch.timer", -120: "evidence_level": "artifact_or_health", -122: "generated_health_output": "/var/lib/homelab-backup-matrix/rendered/homelab-appbackup-npmplus-edge.txt", -123: "id": "homelab_appbackup_npmplus_edge", -DEPENDENCY_FILE_INDEX_END=1 -RELAY_HEALTHCHECK_ARTIFACT_BEGIN=1 -ARTIFACT_MISSING=/usr/local/sbin/pvepro-relay-healthcheck -ARTIFACT_MISSING=/etc/systemd/system/pvepro-relay-healthcheck.service -ARTIFACT_MISSING=/etc/systemd/system/pvepro-relay-healthcheck.timer -ARTIFACT_MISSING=/etc/systemd/system/pvepro-relay-healthcheck.service.d/20-gotify-notifier.conf -RELAY_HEALTHCHECK_ARTIFACT_END=1 -NETBIRD_STATE_BEGIN=1 -NETBIRD_STATUS_JSON_RC=0 -NETBIRD_JSON_PARSE=true -NETBIRD_DAEMON_STATUS=Connected -NETBIRD_MANAGEMENT_CONNECTED=false -NETBIRD_SIGNAL_CONNECTED=false -NETBIRD_RELAYS_TOTAL=2 -NETBIRD_RELAYS_AVAILABLE=0 -NETBIRD_PEER|fqdn=pve02.netbird.selfhosted|status=Connected|connection=P2P -NETBIRD_PEER|fqdn=home-pc.netbird.selfhosted|status=Connecting|connection=- -NETBIRD_PEER|fqdn=relay.netbird.selfhosted|status=Connecting|connection=- -NETBIRD_PEER|fqdn=pve03.netbird.selfhosted|status=Connected|connection=P2P -NETBIRD_PEER|fqdn=edge-vm.netbird.selfhosted|status=Connecting|connection=- -NETBIRD_PEER|fqdn=desktop-moy.netbird.selfhosted|status=Connecting|connection=- -NETBIRD_PEER|fqdn=comp01.netbird.selfhosted|status=Connecting|connection=- -NETBIRD_PEER|fqdn=e3qxxx.netbird.selfhosted|status=Connecting|connection=- -NETBIRD_PEER|fqdn=mail.netbird.selfhosted|status=Connecting|connection=- -NETBIRD_PEER|fqdn=desktop-server.netbird.selfhosted|status=Connecting|connection=- -NETBIRD_PEER|fqdn=e3qxxx-183-106.netbird.selfhosted|status=Connecting|connection=- -NETBIRD_NETWORKS_LIST_RC=0 -No networks available. -NETBIRD_ROUTES_LIST_RC=0 -No networks available. -NETBIRD_STATE_END=1 -PROCESS_AND_SOCKET_REFERENCES_BEGIN=1 - 1220 netbird /usr/bin/netbird service run --log-level info --daemon-addr unix:///var/run/netbird.sock --log-file /var/log/netbird/client.log -ESTAB 0 1500 [IP]:33490 [IP]:443 users:(("netbird",pid=1220,fd=35)) -ESTAB 0 1500 [IP]:33632 [IP]:443 users:(("netbird",pid=1220,fd=18)) -ESTAB 0 1500 [IP]:34548 [IP]:443 users:(("netbird",pid=1220,fd=43)) -ESTAB 0 1500 [IP]:37588 [IP]:443 users:(("netbird",pid=1220,fd=19)) -ESTAB 0 1500 [IP]:39034 [IP]:443 users:(("netbird",pid=1220,fd=20)) -ESTAB 0 1500 [IP]:39134 [IP]:443 users:(("netbird",pid=1220,fd=40)) -ESTAB 0 1500 [IP]:40402 [IP]:443 users:(("netbird",pid=1220,fd=11)) -ESTAB 0 1500 [IP]:41518 [IP]:443 users:(("netbird",pid=1220,fd=33)) -ESTAB 0 1500 [IP]:43330 [IP]:443 users:(("netbird",pid=1220,fd=17)) -ESTAB 0 1500 [IP]:43952 [IP]:443 users:(("netbird",pid=1220,fd=31)) -ESTAB 0 1500 [IP]:44564 [IP]:443 users:(("netbird",pid=1220,fd=41)) -ESTAB 0 1500 [IP]:47572 [IP]:443 users:(("netbird",pid=1220,fd=37)) -ESTAB 0 1500 [IP]:48242 [IP]:443 users:(("netbird",pid=1220,fd=30)) -ESTAB 0 0 [IP]:22 [IP]:61241 users:(("sshd-session",pid=2731736,fd=7),("sshd-session",pid=2731520,fd=7)) -ESTAB 0 1500 [IP]:50512 [IP]:443 users:(("netbird",pid=1220,fd=21)) -ESTAB 0 1500 [IP]:52460 [IP]:443 users:(("netbird",pid=1220,fd=16)) -ESTAB 0 1500 [IP]:53972 [IP]:443 users:(("netbird",pid=1220,fd=38)) -ESTAB 0 1500 [IP]:59556 [IP]:443 users:(("netbird",pid=1220,fd=39)) -PROCESS_AND_SOCKET_REFERENCES_END=1 -DNS_AND_HOST_REFERENCES_BEGIN=1 -DNS_FILE=/etc/hosts|sha256=c3b9101ad96911d55328b70e12e87f9b17e4ef89fce47dc619478e113c4855ec -11:[IP] pve01.gram1.ru pve01 -12:[IP] pve02.gram1.ru pve02 -13:[IP] pve03.gram1.ru pve03 -DNS_FILE=/etc/resolv.conf|sha256=be701c9795e17f831d600a2ad82e50a15785fb48a21d105d6413bb5b98a79fef -1:# Generated by NetBird -2:# The original file can be restored from /etc/resolv.conf.original.netbird -4:search netbird.selfhosted gram1.ru -DNS_AND_HOST_REFERENCES_END=1 -INTERPRETATION_RULES_BEGIN=1 -RULE_1=DO_NOT_DELETE_PROVIDER_VPS_WHILE_MD_RELAY_BACKUP_OR_MONITORING_REFERENCE_IS_ACTIVE_OR_UNRECONCILED -RULE_2=DO_NOT_DELETE_WHILE_ANY_NETBIRD_ROUTE_OR_EXIT_NETWORK_STILL_SELECTS_MOLDOVA_FOR_AN_ACTIVE_PEER -RULE_3=DO_NOT_DELETE_WHILE_PVEPRO_RELAY_HEALTHCHECK_OR_ALERTING_EXPECTS_THIS_NODE_UNLESS_REPLACED_OR_RETIRED -RULE_4=DO_NOT_DELETE_WHILE_BACKUP_AUDIT_IS_FAILED_UNLESS_FAILURE_IS_PROVEN_UNRELATED_AND_BASELINE_IS_RESTORED -RULE_5=AFTER_DETACH_USE_A_REVERSIBLE_QUARANTINE_WINDOW_BEFORE_PROVIDER_DELETION -INTERPRETATION_RULES_END=1 -DECISION=HOLD_DELETE_PENDING_EXACT_PREFLIGHT_REVIEW -NEXT_GATE=READ_EXACT_1130R1_AND_BUILD_NARROW_DETACH_OR_QUARANTINE_PLAN -TASK_RESULT=PASS_MOLDOVA_RETIRE_PREFLIGHT_EVIDENCE_COLLECTED -WORKERS2_MOLDOVA_RETIRE_PREFLIGHT_END=1 +COMPONENT=forum-pve02-vm160-audit +MUTATION_BOUNDARY=NONE;PVE02_VM160_READ_ONLY;NO_MAIL_SEND;NO_DB_WRITE;NO_SERVICE_RESTART;NO_VM_CONFIG;NO_CLOUD +EXECUTION_STARTED=true +CHANGES_MADE=false +ROLLBACK_STARTED=false +ROLLBACK_RESTORED=null +HANDOFF_SHA256=82a6060ccc216849c55790a3489ef157d5551a0c7cef0c5ec9abd35f569c7940 +PRIOR_CID=FORUM-260923-RCA-DEEP-AUDIT8D42R1 +PRIOR_FAILURE=VM160_NOT_RUNNING_ON_PVE01_WRONG_NODE +CORRECTION=USE_PVE02_OWNER_NODE +PVE02_TRANSPORT=PASS +PVE02_EXPLICIT_KEY_USED=false +PVE02_REMOTE_BEGIN=true +PVE02_HOST=pve02 +VM160_STATE=running +VM160_QGA_PING=OK +VM160_QGA_TRANSPORT_RC=0 +VM160_GUEST_EXITCODE=0 +VM160_GUEST_AUDIT_BEGIN=true +HOSTNAME=forum-prod +TIME_UTC=2026-09-23T21:02:35Z +=== SERVICES === +UNIT=nginx ACTIVE=active ENABLED=enabled +UNIT=mariadb ACTIVE=active ENABLED=enabled +UNIT=mysql ACTIVE=active ENABLED=alias +UNIT=php8.3-fpm ACTIVE=active ENABLED=enabled +UNIT=cron ACTIVE=active ENABLED=enabled +=== FAILED UNITS === +=== CONFIG TESTS === +NGINX_CONFIGTEST_RC=0 +nginx: the configuration file /etc/nginx/nginx.conf syntax is ok +nginx: configuration file /etc/nginx/nginx.conf test is successful +PHPFPM_CONFIGTEST_BIN=php-fpm8.3 RC=0 +[23-Sep-2026 21:02:36] NOTICE: configuration file /etc/php/8.3/fpm/php-fpm.conf test is successful +=== SENDMAIL/MSMTP STATE === +MAIL_BIN=/usr/sbin/sendmail mode=755 owner=root group=root size=143208 +MAIL_LINK=/usr/sbin/sendmail TARGET=../bin/msmtp +MAIL_BIN=/usr/bin/msmtp mode=755 owner=root group=root size=143208 +MAIL_BIN_SHA256=d86ba2c0e12187ba16dcac322e4e94bd6079e635c1793b1690678b45bf3d0e61 PATH=/usr/bin/msmtp +MAIL_BIN=/usr/local/sbin/codevipe-sendmail mode=755 owner=root group=root size=810 +MAIL_BIN_SHA256=e50ba5dcf4ac7a905ed1e6b1cc7c301a61318f9fe73976eeda3dd21047f39958 PATH=/usr/local/sbin/codevipe-sendmail +MSMTP_CONFIG_META=/etc/msmtprc mode=640 owner=root group=xfp_mail size=319 +MSMTP_CONFIG_SHA256=c00fe15969542c6cda983e9b937488c9745e97e8fd8b2d75fdf6bd5386e722a9 PATH=/etc/msmtprc +MSMTP_CONFIG_MISSING=/root/.msmtprc +MSMTP_FILE_META=/etc/msmtp/pvepro.env mode=640 owner=root group=www-data size=150 +MSMTP_FILE_SHA256=32330a022cb6fbde19efb486c9945a6251070a544443a2947e7b222401a1ffee PATH=/etc/msmtp/pvepro.env +sendmail_path => /usr/sbin/sendmail -t -i => /usr/sbin/sendmail -t -i +/etc/php/8.3/fpm/pool.d/xf-dsmods.conf:2:php_admin_value[sendmail_path] = /usr/local/sbin/dsmods-sendmail -t -i +/etc/php/8.3/fpm/pool.d/xf-kingofwolk.conf:2:php_admin_value[sendmail_path] = /usr/local/sbin/kingofwolk-sendmail -t -i +/etc/php/8.3/fpm/pool.d/xf-codevipe.conf:2:php_admin_value[sendmail_path] = /usr/local/sbin/codevipe-sendmail -t -i +/etc/php/8.3/fpm/pool.d/www.conf:486:;php_admin_value[sendmail_path] = /usr/sbin/sendmail -t -i -f +/etc/php/8.3/fpm/pool.d/xf-zakrutim.conf:2:php_admin_value[sendmail_path] = /usr/local/sbin/zakrutim-sendmail -t -i +/etc/php/8.3/fpm/pool.d/xf-hkmods.conf:2:php_admin_value[sendmail_path] = /usr/local/sbin/hkmods-sendmail -t -i +/etc/php/8.3/fpm/pool.d/xf-gamevipe.conf:2:php_admin_value[sendmail_path] = /usr/local/sbin/gamevipe-sendmail -t -i +=== MSMTP RECENT ERRORS === +MSMTP_LOG_PRESENT=/var/log/msmtp.log +Jul 01 07:19:46 host=mail.pvepro.ru tls=on auth=on user= from= recipients= smtpstatus=535 smtpmsg='535 5.7.8 Error: authentication failed: (reason unavailable)' errormsg='authentication failed (method PLAIN)' exitcode=EX_NOPERM +Jul 01 07:23:11 host=mail.pvepro.ru tls=on auth=on user= from= recipients= smtpstatus=535 smtpmsg='535 5.7.8 Error: authentication failed: (reason unavailable)' errormsg='authentication failed (method PLAIN)' exitcode=EX_NOPERM +Jul 01 07:23:19 host=mail.pvepro.ru tls=on auth=on user= from= recipients= smtpstatus=535 smtpmsg='535 5.7.8 Error: authentication failed: (reason unavailable)' errormsg='authentication failed (method PLAIN)' exitcode=EX_NOPERM +Jul 01 07:23:27 host=mail.pvepro.ru tls=on auth=on user= from= recipients= smtpstatus=535 smtpmsg='535 5.7.8 Error: authentication failed: (reason unavailable)' errormsg='authentication failed (method PLAIN)' exitcode=EX_NOPERM +Jul 01 08:08:27 host=mail.pvepro.ru tls=on auth=on user= from= recipients= mailsize=242 smtpstatus=250 smtpmsg='250 2.0.0 Ok: queued as 56D68A7F1C' exitcode=EX_OK +Jul 01 08:59:09 host=mail.pvepro.ru tls=on auth=on user= from= recipients= mailsize=1577 smtpstatus=250 smtpmsg='250 2.0.0 Ok: queued as 78667A73C3' exitcode=EX_OK +Jul 01 09:14:24 host=mail.pvepro.ru tls=on auth=on user= from= recipients= mailsize=1628 smtpstatus=250 smtpmsg='250 2.0.0 Ok: queued as D36DBA756C' exitcode=EX_OK +Jul 12 09:30:36 host=mail.pvepro.ru tls=on auth=on user= from= recipients= mailsize=7695 smtpstatus=250 smtpmsg='250 2.0.0 Ok: queued as 1166CA7FEF' exitcode=EX_OK +Jul 12 09:31:50 host=mail.pvepro.ru tls=on auth=on user= from= recipients= mailsize=7695 smtpstatus=250 smtpmsg='250 2.0.0 Ok: queued as 839D2A7FEF' exitcode=EX_OK +Jul 16 14:13:27 host=mail.pvepro.ru tls=on auth=on user= from= recipients= mailsize=7695 smtpstatus=250 smtpmsg='250 2.0.0 Ok: queued as 8E3D8A7FF9' exitcode=EX_OK +Jul 16 14:14:37 host=mail.pvepro.ru tls=on auth=on user= from= recipients= mailsize=7695 smtpstatus=250 smtpmsg='250 2.0.0 Ok: queued as 839E3A73F4' exitcode=EX_OK +Jul 16 14:15:19 host=mail.pvepro.ru tls=on auth=on user= from= recipients= mailsize=7695 smtpstatus=250 smtpmsg='250 2.0.0 Ok: queued as D1B4BA8012' exitcode=EX_OK +Jul 19 21:07:23 host=mail.pvepro.ru tls=on auth=on user= from= recipients= mailsize=7701 smtpstatus=250 smtpmsg='250 2.0.0 Ok: queued as A6FB3A8121' exitcode=EX_OK +Jul 22 11:29:09 host=mail.pvepro.ru tls=on auth=on user= from= recipients= mailsize=7707 smtpstatus=250 smtpmsg='250 2.0.0 Ok: queued as 77305A8015' exitcode=EX_OK +Jul 24 05:51:06 host=mail.pvepro.ru tls=on auth=on user= from= recipients= mailsize=7681 smtpstatus=250 smtpmsg='250 2.0.0 Ok: queued as ED00EA734F' exitcode=EX_OK +Jul 26 06:38:28 host=mail.pvepro.ru tls=on auth=on user= from= recipients= mailsize=7744 smtpstatus=250 smtpmsg='250 2.0.0 Ok: queued as 2DEA8A7F61' exitcode=EX_OK +Jul 28 19:04:01 host=mail.pvepro.ru tls=on auth=on user= from= recipients= mailsize=7697 smtpstatus=250 smtpmsg='250 2.0.0 Ok: queued as 5CE98A737C' exitcode=EX_OK +Jul 28 19:05:08 host=mail.pvepro.ru tls=on auth=on user= from= recipients= mailsize=7697 smtpstatus=250 smtpmsg='250 2.0.0 Ok: queued as E140AA737C' exitcode=EX_OK +Jul 29 08:48:58 host=mail.pvepro.ru tls=on auth=on user= from= recipients= mailsize=7669 smtpstatus=250 smtpmsg='250 2.0.0 Ok: queued as 227FBA7431' exitcode=EX_OK +Jul 29 20:33:47 host=mail.pvepro.ru tls=on auth=on user= from= recipients= mailsize=7685 smtpstatus=250 smtpmsg='250 2.0.0 Ok: queued as 9EBCFA7F0E' exitcode=EX_OK +Jul 30 16:48:00 host=mail.pvepro.ru tls=on auth=on user= from= recipients= mailsize=7687 smtpstatus=250 smtpmsg='250 2.0.0 Ok: queued as B1345A737C' exitcode=EX_OK +Aug 17 08:05:32 host=mail.pvepro.ru tls=on auth=on user= from= recipients= mailsize=7738 smtpstatus=250 smtpmsg='250 2.0.0 Ok: queued as 3EF03A7F8D' exitcode=EX_OK +Aug 17 08:07:52 host=mail.pvepro.ru tls=on auth=on user= from= recipients= mailsize=7738 smtpstatus=250 smtpmsg='250 2.0.0 Ok: queued as 6635BA73AA' exitcode=EX_OK +Aug 18 16:58:38 host=postbox.cloud.yandex.net tls=on auth=on user=ajeups2v3l2qfl9t1k9a from= recipients= mailsize=440 smtpstatus=250 smtpmsg='250 2.6.0 OK: queued as DKS89L8IPX05.3NYOOU5KRIBAX@ingress2-vla' exitcode=EX_OK +Aug 18 18:59:07 host=postbox.cloud.yandex.net tls=on auth=on user=ajeups2v3l2qfl9t1k9a from= recipients= mailsize=492 smtpstatus=250 smtpmsg='250 2.6.0 OK: queued as DKSATUF4C1XX.329CV6B95FH4R@ingress2-vla' exitcode=EX_OK +Aug 21 15:02:48 host=postbox.cloud.yandex.net tls=on auth=on user=ajeups2v3l2qfl9t1k9a from= recipients= mailsize=7672 smtpstatus=250 smtpmsg='250 2.6.0 OK: queued as DKUPOJOWNNPA.3CH5IJH2A8PW1@ingress2-klg' exitcode=EX_OK +Sep 02 19:39:11 host=postbox.cloud.yandex.net tls=on auth=on user=ajeups2v3l2qfl9t1k9a from= recipients= smtpstatus=550 smtpmsg='550 5.4.1 identity not found RequestID: 488b75b4-c264-42eb-b6fc-0cae6b175a7a' errormsg='envelope from address not accepted by the server' exitcode=EX_DATAERR +Sep 02 19:43:53 host=smtp.yandex.ru tls=on auth=on user= from= recipients= mailsize=257 smtpstatus=250 smtpmsg='250 2.0.0 Ok: queued on mail-nwsmtp-smtp-production-main-95.klg.yp-c.yandex.net 1788378233-qhqjKj2mO8c0-EjXDwBNJ' exitcode=EX_OK +Sep 02 19:44:53 host=smtp.yandex.ru tls=on auth=on user= from= recipients= mailsize=254 smtpstatus=250 smtpmsg='250 2.0.0 Ok: queued on mail-nwsmtp-smtp-production-main-71.sas.yp-c.yandex.net 1788378293-riqme1Oc6uQ0-JA3QSjr3' exitcode=EX_OK +Sep 02 21:33:45 host=smtp.yandex.ru tls=on auth=on user= from= recipients= mailsize=7477 smtpstatus=250 smtpmsg='250 2.0.0 Ok: queued on mail-nwsmtp-smtp-production-main-99.klg.yp-c.yandex.net 1788384825-iXsB0HPeHeA0-t67v0c58' exitcode=EX_OK +Sep 03 04:35:14 host=smtp.yandex.ru tls=on auth=on user= from= recipients= mailsize=2390 smtpstatus=250 smtpmsg='250 2.0.0 Ok: queued on mail-nwsmtp-smtp-production-main-85.iva.yp-c.yandex.net 1788410114-DZclo65g6Sw0-ym39kyd6' exitcode=EX_OK +Sep 03 04:44:35 host=smtp.yandex.ru tls=on auth=on user= from= recipients= mailsize=2412 smtpstatus=250 smtpmsg='250 2.0.0 Ok: queued on mail-nwsmtp-smtp-production-main-89.sas.yp-c.yandex.net 1788410675-ZicUTjSgH4Y0-ino0SiK0' exitcode=EX_OK +Sep 04 04:09:17 host=smtp.yandex.ru tls=on auth=on user= from= recipients= mailsize=7474 smtpstatus=250 smtpmsg='250 2.0.0 Ok: queued on mail-nwsmtp-smtp-production-main-88.klg.yp-c.yandex.net 1788494957-H9Anioed9mI0-YAPrVYsC' exitcode=EX_OK +Sep 04 04:09:44 host=smtp.yandex.ru tls=on auth=on user= from= recipients= mailsize=20404 smtpstatus=250 smtpmsg='250 2.0.0 Ok: queued on mail-nwsmtp-smtp-production-main-59.klg.yp-c.yandex.net 1788494984-h9AZQbdd08c0-OXrP5vgQ' exitcode=EX_OK +Sep 04 04:19:31 host=smtp.yandex.ru tls=on auth=on user= from= recipients= mailsize=7632 smtpstatus=250 smtpmsg='250 2.0.0 Ok: queued on mail-nwsmtp-smtp-production-main-98.sas.yp-c.yandex.net 1788495571-UJAjYwgdJCg0-xBGi7vHN' exitcode=EX_OK +Sep 04 06:00:34 host=smtp.yandex.ru tls=on auth=on user= from= recipients= mailsize=7703 smtpstatus=250 smtpmsg='250 2.0.0 Ok: queued on mail-nwsmtp-smtp-production-main-59.iva.yp-c.yandex.net 1788501634-X0CkCDGeJqM0-ewqs8N37' exitcode=EX_OK +Sep 08 08:25:56 host=smtp.yandex.ru tls=on auth=on user= from= recipients= mailsize=7704 smtpstatus=250 smtpmsg='250 2.0.0 Ok: queued on mail-nwsmtp-smtp-production-main-89.sas.yp-c.yandex.net 1788855956-tPIpFURgCSw0-0bD0dt06' exitcode=EX_OK +Sep 10 00:10:02 host=smtp.yandex.ru tls=on auth=on user= from= recipients= smtpstatus=553 smtpmsg='553 5.7.1 Sender address rejected: user not found. 1788999002-1ACLU4egAmI0-BvLQE2bE' errormsg='envelope from address not accepted by the server' exitcode=EX_DATAERR +Sep 10 00:10:03 host=smtp.yandex.ru tls=on auth=on user= from= recipients= mailsize=10460 smtpstatus=250 smtpmsg='250 2.0.0 Ok: queued on mail-nwsmtp-smtp-production-main-71.sas.yp-c.yandex.net 1788999003-3ACiEWkcAmI0-sgTWLJGA' exitcode=EX_OK +Sep 10 00:10:05 host=smtp.yandex.ru tls=on auth=on user= from= recipients= mailsize=10599 smtpstatus=250 smtpmsg='250 2.0.0 Ok: queued on mail-nwsmtp-smtp-production-main-60.sas.yp-c.yandex.net 1788999005-4ACnvojfQa60-IhiFG02U' exitcode=EX_OK +Sep 10 00:10:06 host=smtp.yandex.ru tls=on auth=on user= from= recipients= smtpstatus=553 smtpmsg='553 5.7.1 Sender address rejected: user not found. 1788999006-6ACgTJhfxuQ0-ykq2gT2m' errormsg='envelope from address not accepted by the server' exitcode=EX_DATAERR +Sep 10 00:10:07 host=smtp.yandex.ru tls=on auth=on user= from= recipients= smtpstatus=553 smtpmsg='553 5.7.1 Sender address rejected: user not found. 1788999007-7ACYnudg6mI0-wrzZ9ta2' errormsg='envelope from address not accepted by the server' exitcode=EX_DATAERR +Sep 10 00:10:08 host=smtp.yandex.ru tls=on auth=on user= from= recipients= mailsize=10224 smtpstatus=250 smtpmsg='250 2.0.0 Ok: queued on mail-nwsmtp-smtp-production-main-58.iva.yp-c.yandex.net 1788999008-8ACm4u3oe8c0-EAVVXjUV' exitcode=EX_OK +Sep 13 16:14:26 host=smtp.yandex.ru tls=on auth=on user= from= recipients= mailsize=7702 smtpstatus=250 smtpmsg='250 2.0.0 Ok: queued on mail-nwsmtp-smtp-production-main-67.vla.yp-c.yandex.net 1789316066-QEVwdXHe9Os0-PH6Ad0od' exitcode=EX_OK +Sep 19 00:10:02 host=smtp.yandex.ru tls=on auth=on user= from= recipients= smtpstatus=553 smtpmsg='553 5.7.1 Sender address rejected: user not found. 1789776602-2ALqagFfpGk0-vq5tlHNF' errormsg='envelope from address not accepted by the server' exitcode=EX_DATAERR +Sep 19 00:10:03 host=smtp.yandex.ru tls=on auth=on user= from= recipients= smtpstatus=553 smtpmsg='553 5.7.1 Sender address rejected: user not found. 1789776603-3ALZ9cFg74Y0-Rbt8yJgu' errormsg='envelope from address not accepted by the server' exitcode=EX_DATAERR +Sep 19 00:10:04 host=smtp.yandex.ru tls=on auth=on user= from= recipients= smtpstatus=553 smtpmsg='553 5.7.1 Sender address rejected: user not found. 1789776604-4AL5ABFdSSw0-XwXfZVbq' errormsg='envelope from address not accepted by the server' exitcode=EX_DATAERR +Sep 19 00:10:05 host=smtp.yandex.ru tls=on auth=on user= from= recipients= smtpstatus=553 smtpmsg='553 5.7.1 Sender address rejected: user not found. 1789776605-5ALSb5FdeKo0-ugaNiHmf' errormsg='envelope from address not accepted by the server' exitcode=EX_DATAERR +Sep 19 00:10:06 host=smtp.yandex.ru tls=on auth=on user= from= recipients= smtpstatus=553 smtpmsg='553 5.7.1 Sender address rejected: user not found. 1789776606-6ALAYdFdNuQ0-dL2d7DHS' errormsg='envelope from address not accepted by the server' exitcode=EX_DATAERR +Sep 19 00:10:07 host=smtp.yandex.ru tls=on auth=on user= from= recipients= smtpstatus=553 smtpmsg='553 5.7.1 Sender address rejected: user not found. 1789776607-7ALu7bFe4Gk0-WcjF4jNs' errormsg='envelope from address not accepted by the server' exitcode=EX_DATAERR +=== JOB RUNNER === +JOBRUNNER_META=/root/scripts/forum-xenforo-run-jobs.sh mode=700 owner=root group=root size=2343 +JOBRUNNER_SHA256=5cc6c6cba317e0f0c4b1b13f375d8677b3c9c96d0416f998315a7d4e05e8a1aa PATH=/root/scripts/forum-xenforo-run-jobs.sh +JOBRUNNER_META=/etc/cron.d/forum-xenforo-run-jobs mode=644 owner=root group=root size=183 +JOBRUNNER_SHA256=20cf9e4d059148c14094d145e778bb307b21183f608ff9b430dd620b9ece75c4 PATH=/etc/cron.d/forum-xenforo-run-jobs +=== XENFORO SITES === +XENFORO_ROOT_COUNT=6 +SITE_BEGIN=1 +ROOT_SHA256=90d10058f0574d8a5ff1ff412fb7b6a1eab2119d09c8b8162a0f8ca55d15180f +DB_CONNECT=OK +BOARD_DOMAIN=codevipe.ru +EMAIL_TRANSPORT={"emailTransport":"sendmail"} +DEFAULTEMAILADDRESS_PRESENT=true +DEFAULTEMAILADDRESS_SHA256=2d4e7d3cf84ec1ba079400cf62ca99e7464a221a686ebe73405b73fb01ffe6f4 +BOUNCEEMAILADDRESS_PRESENT=true +BOUNCEEMAILADDRESS_SHA256=1b5fc8a44aa74c2f02b7ce893f8a0e0a901d60b3f5e68b39c9448e7265117265 +CONTACTEMAILADDRESS_PRESENT=true +CONTACTEMAILADDRESS_SHA256=0d4cf60ca8cd53a12ad50dc5419d3c9c32192f0852000eb10051fbdef52b04bf +XF_ERROR_COUNT_24H=0 +XF_ERROR_COUNT_7D=2 +XF_ERROR_COUNT_30D=2 +XF_ERROR_COUNT_ALL=2 +XF_ERROR_TYPE=XF\Db\InvalidQueryException COUNT=1 LAST=1789960536 +XF_ERROR_TYPE=Symfony\Component\Mailer\Exception\TransportException COUNT=1 LAST=1789776602 +XF_ERROR_ROW ID=2 TS=1789960536 TYPE=XF\Db\InvalidQueryException FILE=AbstractStatement.php LINE=225 MSG=MySQL statement prepare error [1064]: You have an error in your SQL syntax; check the manual that corresponds to your MariaDB server version for the right syntax to use near 'FROM xf_template_modification WHERE addon_id='Gram/Support' ORDER MSG_SHA256=9ad3a33fd2ebdd213c22e2156d0358b02f9f6f82e2027f3101055bbfe5dd83b5 +XF_ERROR_ROW ID=1 TS=1789776602 TYPE=Symfony\Component\Mailer\Exception\TransportException FILE=ProcessStream.php LINE=74 MSG=Email to from failed: Process failed with exit code 65: sendmail: envelope from address not accepted by the server sendmail: server message: 553 5.7.1 Sender address rejected: user not found. 1789776602-2ALqagFfpGk0- MSG_SHA256=6ec17ccf28a8f55861f6ceb88ab722165bd922e90a12976f2755bc9db10c3638 +XF_MAIL_QUEUE_TABLE=MISSING +XF_JOB_COUNT=2 +SITE_PHP_AUDIT_RC=0 +SITE_END=1 +SITE_BEGIN=2 +ROOT_SHA256=d33b4c353c031284c61f9632f49971cd1f833ab4f48c0da27e4fdf81c373652e +DB_CONNECT=OK +BOARD_DOMAIN=dsmods.ru +EMAIL_TRANSPORT={"emailTransport":"sendmail"} +DEFAULTEMAILADDRESS_PRESENT=true +DEFAULTEMAILADDRESS_SHA256=953017f372fc160381792c8f72822095c821e5f0c4519776c44cfa01fb6c16f1 +BOUNCEEMAILADDRESS_PRESENT=true +BOUNCEEMAILADDRESS_SHA256=ba70f629ed27c307125a0538ca1c95049ac153aea2b21a474afeb4e3ad60d08d +CONTACTEMAILADDRESS_PRESENT=true +CONTACTEMAILADDRESS_SHA256=0d4cf60ca8cd53a12ad50dc5419d3c9c32192f0852000eb10051fbdef52b04bf +XF_ERROR_COUNT_24H=0 +XF_ERROR_COUNT_7D=2 +XF_ERROR_COUNT_30D=2 +XF_ERROR_COUNT_ALL=2 +XF_ERROR_TYPE=XF\Db\InvalidQueryException COUNT=1 LAST=1789960537 +XF_ERROR_TYPE=Symfony\Component\Mailer\Exception\TransportException COUNT=1 LAST=1789776606 +XF_ERROR_ROW ID=2 TS=1789960537 TYPE=XF\Db\InvalidQueryException FILE=AbstractStatement.php LINE=225 MSG=MySQL statement prepare error [1064]: You have an error in your SQL syntax; check the manual that corresponds to your MariaDB server version for the right syntax to use near 'FROM xf_template_modification WHERE addon_id='Gram/Support' ORDER MSG_SHA256=9ad3a33fd2ebdd213c22e2156d0358b02f9f6f82e2027f3101055bbfe5dd83b5 +XF_ERROR_ROW ID=1 TS=1789776606 TYPE=Symfony\Component\Mailer\Exception\TransportException FILE=ProcessStream.php LINE=74 MSG=Email to from failed: Process failed with exit code 65: sendmail: envelope from address not accepted by the server sendmail: server message: 553 5.7.1 Sender address rejected: user not found. 1789776606-6ALAYdFdNuQ0- MSG_SHA256=59e1133715aedc5b530f2a2f87f31d659af13424d86fdc7b06d1331635178a0c +XF_MAIL_QUEUE_TABLE=MISSING +XF_JOB_COUNT=2 +SITE_PHP_AUDIT_RC=0 +SITE_END=2 +SITE_BEGIN=3 +ROOT_SHA256=e06523a33d0b38a0ec6532783b5e63542eeba1303e68f2efad63214fd1e8908f +DB_CONNECT=OK +BOARD_DOMAIN=gamevipe.ru +EMAIL_TRANSPORT={"emailTransport":"sendmail"} +DEFAULTEMAILADDRESS_PRESENT=true +DEFAULTEMAILADDRESS_SHA256=ded102c05ab32e41f2ada4d15b63c6adb848e14e76154015f294b751459a75dc +BOUNCEEMAILADDRESS_PRESENT=true +BOUNCEEMAILADDRESS_SHA256=3c6809f1cc3dd12044ad384b8d944a408bd84b4320af0bb505a5fb5ebf3f10e3 +CONTACTEMAILADDRESS_PRESENT=true +CONTACTEMAILADDRESS_SHA256=0d4cf60ca8cd53a12ad50dc5419d3c9c32192f0852000eb10051fbdef52b04bf +XF_ERROR_COUNT_24H=0 +XF_ERROR_COUNT_7D=1 +XF_ERROR_COUNT_30D=1 +XF_ERROR_COUNT_ALL=1 +XF_ERROR_TYPE=Symfony\Component\Mailer\Exception\TransportException COUNT=1 LAST=1789776603 +XF_ERROR_ROW ID=1 TS=1789776603 TYPE=Symfony\Component\Mailer\Exception\TransportException FILE=ProcessStream.php LINE=74 MSG=Email to from failed: Process failed with exit code 65: sendmail: envelope from address not accepted by the server sendmail: server message: 553 5.7.1 Sender address rejected: user not found. 1789776603-3ALZ9cFg74Y0- MSG_SHA256=f0aeb49dcba2c41db1f69663611a12e57913ca0b631287cdcc3c3b2a8db73f51 +XF_MAIL_QUEUE_TABLE=MISSING +XF_JOB_COUNT=3 +SITE_PHP_AUDIT_RC=0 +SITE_END=3 +SITE_BEGIN=4 +ROOT_SHA256=127c45667b0b6b6f23d20d6cc66d094b61a854801e42025ad2cb81ab5bfbc03e +DB_CONNECT=OK +BOARD_DOMAIN=hkmods.ru +EMAIL_TRANSPORT={"emailTransport":"sendmail"} +DEFAULTEMAILADDRESS_PRESENT=true +DEFAULTEMAILADDRESS_SHA256=ef08c646d1d1fc56ef350fba44e29f2117996462bf46d9d14fa33c0688f9f4e3 +BOUNCEEMAILADDRESS_PRESENT=true +BOUNCEEMAILADDRESS_SHA256=0097bdfb68ae6258a87c12c8dae4823421d4209086a2f190cb189defbda96bc3 +CONTACTEMAILADDRESS_PRESENT=true +CONTACTEMAILADDRESS_SHA256=0d4cf60ca8cd53a12ad50dc5419d3c9c32192f0852000eb10051fbdef52b04bf +XF_ERROR_COUNT_24H=0 +XF_ERROR_COUNT_7D=1 +XF_ERROR_COUNT_30D=1 +XF_ERROR_COUNT_ALL=1 +XF_ERROR_TYPE=Symfony\Component\Mailer\Exception\TransportException COUNT=1 LAST=1789776604 +XF_ERROR_ROW ID=1 TS=1789776604 TYPE=Symfony\Component\Mailer\Exception\TransportException FILE=ProcessStream.php LINE=74 MSG=Email to from failed: Process failed with exit code 65: sendmail: envelope from address not accepted by the server sendmail: server message: 553 5.7.1 Sender address rejected: user not found. 1789776604-4AL5ABFdSSw0- MSG_SHA256=03dcb6ecd92e29dde8dcccc9776bb4af1985d7d525bfae5cf9b5bb4db579de95 +XF_MAIL_QUEUE_TABLE=MISSING +XF_JOB_COUNT=3 +SITE_PHP_AUDIT_RC=0 +SITE_END=4 +SITE_BEGIN=5 +ROOT_SHA256=d249590aad8f87daf95d1fed623405edb307aa6c9b8123b2aaf89da88fc117ab +DB_CONNECT=OK +BOARD_DOMAIN=kingofwolk.ru +EMAIL_TRANSPORT={"emailTransport":"sendmail"} +DEFAULTEMAILADDRESS_PRESENT=true +DEFAULTEMAILADDRESS_SHA256=e3b5d95ee979b3970d8ac7a07e503a6f5247963b701b1d1ad0b4aca12fc41d3a +BOUNCEEMAILADDRESS_PRESENT=true +BOUNCEEMAILADDRESS_SHA256=e49d0b3b9f084cf1223dd2b108019277e2b51caf2cf1a63b3ba3853765aefc65 +CONTACTEMAILADDRESS_PRESENT=true +CONTACTEMAILADDRESS_SHA256=0d4cf60ca8cd53a12ad50dc5419d3c9c32192f0852000eb10051fbdef52b04bf +XF_ERROR_COUNT_24H=0 +XF_ERROR_COUNT_7D=1 +XF_ERROR_COUNT_30D=1 +XF_ERROR_COUNT_ALL=1 +XF_ERROR_TYPE=Symfony\Component\Mailer\Exception\TransportException COUNT=1 LAST=1789776607 +XF_ERROR_ROW ID=1 TS=1789776607 TYPE=Symfony\Component\Mailer\Exception\TransportException FILE=ProcessStream.php LINE=74 MSG=Email to from failed: Process failed with exit code 65: sendmail: envelope from address not accepted by the server sendmail: server message: 553 5.7.1 Sender address rejected: user not found. 1789776607-7ALu7bFe4Gk0- MSG_SHA256=075f2b1963657d083c11e214ecc8cfd4e01d30f9ac1723bc440808730e6fb809 +XF_MAIL_QUEUE_TABLE=MISSING +XF_JOB_COUNT=2 +SITE_PHP_AUDIT_RC=0 +SITE_END=5 +SITE_BEGIN=6 +ROOT_SHA256=b5bc9e40848a038adb66a5656917858172c1b70a14ccbe680e9d8f2260dda046 +DB_CONNECT=OK +BOARD_DOMAIN=zakrutim.ru +EMAIL_TRANSPORT={"emailTransport":"sendmail"} +DEFAULTEMAILADDRESS_PRESENT=true +DEFAULTEMAILADDRESS_SHA256=b719f473d6ff99139fef2749e636437b76aaf482963e82dde4ac69a296a2eab9 +BOUNCEEMAILADDRESS_PRESENT=true +BOUNCEEMAILADDRESS_SHA256=a93aa0f45435a0e2c159c501fbe3cc70deaff82401d29d87a972e633f91df83a +CONTACTEMAILADDRESS_PRESENT=true +CONTACTEMAILADDRESS_SHA256=0d4cf60ca8cd53a12ad50dc5419d3c9c32192f0852000eb10051fbdef52b04bf +XF_ERROR_COUNT_24H=0 +XF_ERROR_COUNT_7D=1 +XF_ERROR_COUNT_30D=1 +XF_ERROR_COUNT_ALL=1 +XF_ERROR_TYPE=Symfony\Component\Mailer\Exception\TransportException COUNT=1 LAST=1789776605 +XF_ERROR_ROW ID=1 TS=1789776605 TYPE=Symfony\Component\Mailer\Exception\TransportException FILE=ProcessStream.php LINE=74 MSG=Email to from failed: Process failed with exit code 65: sendmail: envelope from address not accepted by the server sendmail: server message: 553 5.7.1 Sender address rejected: user not found. 1789776605-5ALSb5FdeKo0- MSG_SHA256=894c6c5636182853711e897fbcf34d5881646135b6dffd7360737376e789d384 +XF_MAIL_QUEUE_TABLE=MISSING +XF_JOB_COUNT=2 +SITE_PHP_AUDIT_RC=0 +SITE_END=6 +=== RECENT SERVICE ERROR COUNTS === +JOURNAL_ERROR_COUNT_24H_nginx=0 +JOURNAL_ERROR_COUNT_24H_mariadb=0 +JOURNAL_ERROR_COUNT_24H_mysql=0 +JOURNAL_ERROR_COUNT_24H_php8_3_fpm=0 +VM160_GUEST_AUDIT_END=true +VM160_GUEST_STDERR_PRESENT=false +VM160_QGA_PARSE_RC=0 +PVE02_REMOTE_END=true +FAILURE=NONE +DECISION=PASS_PVE02_VM160_AUDIT +NEXT_GATE=REVIEW_FOR_BOUNDED_REPAIR +TASK_RESULT=READ_ONLY_PVE02_VM160_FORUM_AUDIT_COMPLETE +HOMELAB_RESULT_CONTRACT={"version":1,"command_id":"FORUM-260923-RCA-PVE02-VM160-AUDITC7A1R1","status":"OK","changes_made":false,"rollback_started":false,"rollback_restored":null} OUTPUT_END CHAT_OUTPUT_END