homelab-runtime-publisher
2114 lines
255 KiB
Plaintext
2114 lines
255 KiB
Plaintext
CHAT_OUTPUT_BEGIN
|
|
COMMAND_ID=SUPPORT-260923-HOMELAB-MOLDOVA-RETIRE-PREFLIGHT-1130R1
|
|
STATUS=OK
|
|
RC=0
|
|
HOST=pve01
|
|
MODE=read-only
|
|
COMPONENT=homelab-moldova-retire-preflight
|
|
REFERENCE_REGISTER_CHECK=OK
|
|
REFERENCE_SHA256=5a3d8e5154c41cb582a4e0aca68090be1f0138918bf82131a948df326f9d8d66
|
|
ERROR_REGISTER_CHECK=OK
|
|
ERROR_REGISTER_SHA256=3b09a553ec0f527ed3afeed4753f52a74ee3036045bbfb3c685e9f8af4ba7ba0
|
|
COMMAND_SHA256=724447a90ca704066487f2c058f6cbf14e3086e7b399e385e5bca6df62d1cd70
|
|
DUPLICATE_FAILED_COMMAND_BLOCKED=false
|
|
EXECUTION_STARTED=true
|
|
CHANGE_DECLARED=false
|
|
RESULT_CONTRACT_VALID=true
|
|
RESULT_CONTRACT_STATUS=NOT_APPLICABLE
|
|
RESULT_CONTRACT_ERROR=NONE
|
|
COMMAND_RC=0
|
|
CHANGES_MADE=false
|
|
ROLLBACK_STARTED=false
|
|
ROLLBACK_RESTORED=null
|
|
MUTATION_OUTCOME=NO_MUTATION
|
|
SANITIZED=yes
|
|
SECRETS_INCLUDED=no
|
|
PRIVATE_ADDRESSES_INCLUDED=no
|
|
RAW_EVIDENCE_SHA256=f75ed52bc7c1eb7c5418acf9a6f3c7d372de8ddb2bf445b6eb507958c8172cf8
|
|
SANITIZED_OUTPUT_SHA256=c0f54cefd55350849427898c5de7f25cde594a78aba7943a1b3599863593f6ac
|
|
OUTPUT_BEGIN
|
|
WORKERS2_MOLDOVA_RETIRE_PREFLIGHT_BEGIN=1
|
|
COMMAND_ID=SUPPORT-260923-HOMELAB-MOLDOVA-RETIRE-PREFLIGHT-1130R1
|
|
MODE=read-only
|
|
COMPONENT=homelab-moldova-retire-preflight
|
|
MUTATION_BOUNDARY=NONE;LOCAL_DEPENDENCY_DISCOVERY_ONLY;NO_PROVIDER_CALL;NO_TARGET_SSH;NO_NETWORK_MUTATION;NO_SYSTEMD_ACTION;NO_DNS_CHANGE;NO_BACKUP_MUTATION;NO_GIT_WRITE;NO_SECRET_VALUE_READOUT
|
|
PREFLIGHT_PURPOSE=PROVE_WHAT_MUST_BE_DETACHED_OR_REPLACED_BEFORE_MOLDOVA_VPS_CAN_BE_QUARANTINED_OR_DELETED
|
|
REFERENCE_SHA256=5a3d8e5154c41cb582a4e0aca68090be1f0138918bf82131a948df326f9d8d66
|
|
ERROR_REGISTER_SHA256=3b09a553ec0f527ed3afeed4753f52a74ee3036045bbfb3c685e9f8af4ba7ba0
|
|
RUNNER_SHA256=b248a4c32c9cc64e5747e7dce6c7fc0a23f5124a77c71ce72e27a81aceae9d2d
|
|
AUTHORITY_GATE=PASS
|
|
FRESH_CONTEXT_BEGIN=1
|
|
NOW_UTC=2026-09-23T20:58:51Z
|
|
HOSTNAME=pve01
|
|
FRESH_CONTEXT_END=1
|
|
HANDOFF_CANDIDATES_BEGIN=1
|
|
HANDOFF_CANDIDATE_COUNT=4
|
|
HANDOFF_FILE=/srv/homelab-ops/kb/current/04_P0_NEXT.md
|
|
HANDOFF_MTIME=2026-08-21 00:06:53.672037607 +0300
|
|
HANDOFF_SHA256=0f6eb8b23382becc1868e8a22318d5b7629184568205bad47c3c38967a346b8f
|
|
5:3. Canonical post-migration CR including NetBird new USA, Postbox/Null MX, Moldova V3/Gotify, old-USA retirement window and dependencies.
|
|
7:5. VM160 backup-state audit.
|
|
12:Do not mix unrelated changes into these gates (e.g. NetBird version upgrade/hostname change during SSH hardening).
|
|
HANDOFF_FILE=/srv/homelab-ops/kb/changes/CR-2026-0087-HANDOFF.md
|
|
HANDOFF_MTIME=2026-08-21 00:06:53.671922428 +0300
|
|
HANDOFF_SHA256=a8d91b33b36e462bf3bb48defd12e121ed06edcb12d2dab3a9be50b2c5cdcb4d
|
|
13:- Scrutiny and crypto chart-health runtime are removed.
|
|
HANDOFF_FILE=/srv/homelab-ops/kb/changes/CR-2026-0086-HANDOFF.md
|
|
HANDOFF_MTIME=2026-08-21 00:06:53.671835127 +0300
|
|
HANDOFF_SHA256=faab79cda5c6b31162b68812fc468359c0b9b94892c8f1e3b31c626d6704264a
|
|
26:3. load regression/error registry;
|
|
68:Failure: clean clone had no Git `user.name` / `user.email`.
|
|
138:V4 service was deployed side-by-side and later verified healthy, but public
|
|
199:- regression registry
|
|
HANDOFF_FILE=/srv/homelab-ops/kb/AI_CONTEXT.md
|
|
HANDOFF_MTIME=2026-08-21 00:06:53.671629955 +0300
|
|
HANDOFF_SHA256=5fc43e34a1f0714ddacf7c00ed5c4bbaf788df3b8407dcfebd1333a23a623acc
|
|
15:- pve01 `[IP]`, operator historical NetBird path `root@[IP]`, MobaXterm key label `eddsa-key-20260617`.
|
|
26:## Mail/forums
|
|
30:## NetBird migration
|
|
32:Production NetBird is new USA `[IP]`, Debian 12, hostname `nb2`, NetBird `0.73.2`. Exact deployment caveat: `/api/health` = 404 and combined `:9000/health` = 503; these are not generic readiness gates for this deployment. Old USA `[IP]` has Mailcow/NetBird server stopped and observer disabled, but remains in rollback window. Permanent new-USA operator SSH hardening is still P0.
|
|
34:## Moldova
|
|
38:## Backups/cloud
|
|
40:Canonical production backup remotes remain Mail A/B. Full SITE-DR is SEALED; do not rerun merely because architecture changed. Cloud.ru bucket `homelab-dr-backup`: versioning **ON** (this supersedes older OFF docs), writer/reader OFF, no real workload. Yandex Object Storage bucket not created; latest verified billing threshold is `-600 RUB`, target 0, support request pending. Bucket creation remains blocked until live 0 proof.
|
|
HANDOFF_CANDIDATES_END=1
|
|
REFERENCE_MOLDOVA_SNIPPETS_BEGIN=1
|
|
4:SUPERSEDES_OLDER_TOPOLOGY_AND_BACKUP_SNAPSHOTS=YES
|
|
23:- VM180 cluster-admin | DESTROYED | backup registry lifecycle=RETIRED_PRESERVE
|
|
24:- CT200 skladchik-mod | DESTROYED | backup registry lifecycle=RETIRED_PRESERVE
|
|
25:- Scrutiny | RETIRED | active service/Podman removed | backup registry lifecycle=RETIRED_PRESERVE | historical recovery evidence preserved
|
|
28:### BACKUP_CURRENT
|
|
29:BACKUP_STATUS=GREEN
|
|
30:BACKUP_REGISTRY_TOTAL=21
|
|
31:BACKUP_ACTIVE_VM=11
|
|
32:BACKUP_RETIRED_VM=2
|
|
33:BACKUP_ACTIVE_APP=4
|
|
34:BACKUP_RETIRED_APP=3
|
|
35:BACKUP_ACTIVE_CONTROL=1
|
|
36:BACKUP_FRESH_TARGETS=15
|
|
37:BACKUP_SOURCE_DRIFT=0
|
|
38:BACKUP_RESUME=0
|
|
39:BACKUP_PAYLOAD_LEAKS=0
|
|
40:BACKUP_VM_QUEUE=inactive/success
|
|
41:BACKUP_APP_QUEUE=inactive/success
|
|
55:- Immich machine-learning runtime recovered; current stock healthcheck passes and server-to-ML /ping returns HTTP 200.
|
|
72:HEALTH_WARN_ERROR_COUNT=14
|
|
74:NOTE forum-prod VMID160 is not in homelab-nightly-all; it has separate local/cloud backup chain
|
|
75:NOTE edge-vm disk scsi1 backup=0 risk must be documented
|
|
77:NOTE edge health WARN/ERROR items should be documented as known current states
|
|
79:HEALTH_NOT_OK
|
|
81:HEALTH_NOT_OK item=backup-restore-coverage-slo.txt line=STATUS=WARN
|
|
82:HEALTH_NOT_OK item=backup-restore-dashboard.txt line=STATUS=WARN TS=2026-06-29T18:10:09Z ITEMS=80 NOT_OK=13
|
|
83:HEALTH_NOT_OK item=container-image-lifecycle-slo.txt line=STATUS=WARN
|
|
84:HEALTH_NOT_OK item=cron-job-monitoring-slo.txt line=STATUS=WARN
|
|
85:HEALTH_NOT_OK item=direct-heartbeat-pilot-readiness.txt line=STATUS=WARN
|
|
86:HEALTH_NOT_OK item=healthchecks-heartbeat-coverage.txt line=STATUS=WARN
|
|
87:HEALTH_NOT_OK item=healthchecks-job-coverage.txt line=STATUS=WARN
|
|
88:HEALTH_NOT_OK item=healthchecks-job-monitoring-slo.txt line=STATUS=WARN
|
|
89:HEALTH_NOT_OK item=homepage-service-catalog-slo.txt line=STATUS=WARN
|
|
90:HEALTH_NOT_OK item=ingress-dns-route-slo.txt line=STATUS=WARN
|
|
91:HEALTH_NOT_OK item=job-monitoring-slo.txt line=STATUS=WARN
|
|
92:HEALTH_NOT_OK item=observability-health-surface-slo.txt line=STATUS=WARN
|
|
REFERENCE_MOLDOVA_SNIPPETS_END=1
|
|
SYSTEMD_STATE_BEGIN=1
|
|
UNIT|name=pvepro-relay-healthcheck.timer|load=not-found|active=inactive|sub=dead|result=success|enabled=not-found
|
|
UNIT|name=pvepro-relay-healthcheck.service|load=not-found|active=inactive|sub=dead|result=success|enabled=not-found
|
|
UNIT|name=homelab-backup-v2.timer|load=loaded|active=active|sub=waiting|result=success|enabled=enabled
|
|
UNIT|name=homelab-backup-v2.service|load=loaded|active=inactive|sub=dead|result=success|enabled=static
|
|
UNIT|name=homelab-backup-audit.timer|load=loaded|active=active|sub=waiting|result=success|enabled=enabled
|
|
UNIT|name=homelab-backup-audit.service|load=loaded|active=failed|sub=failed|result=exit-code|enabled=static
|
|
UNIT|name=homelab-alertmanager-backup-health.timer|load=loaded|active=active|sub=waiting|result=success|enabled=enabled
|
|
UNIT|name=homelab-alertmanager-backup-health.service|load=loaded|active=inactive|sub=dead|result=success|enabled=static
|
|
UNIT|name=netbird-peers-health.timer|load=loaded|active=active|sub=waiting|result=success|enabled=enabled
|
|
UNIT|name=netbird-peers-health.service|load=loaded|active=failed|sub=failed|result=exit-code|enabled=static
|
|
UNIT|name=netbird.service|load=loaded|active=active|sub=running|result=success|enabled=enabled
|
|
UNIT|name=netbird-vps-backup.timer|load=not-found|active=inactive|sub=dead|result=success|enabled=not-found
|
|
UNIT|name=netbird-vps-backup.service|load=not-found|active=inactive|sub=dead|result=success|enabled=not-found
|
|
UNIT|name=netbird-vps-offhost-sync.timer|load=not-found|active=inactive|sub=dead|result=success|enabled=not-found
|
|
UNIT|name=netbird-vps-offhost-sync.service|load=not-found|active=inactive|sub=dead|result=success|enabled=not-found
|
|
UNIT|name=homelab-crowdsec-capi-netbird-egress.service|load=not-found|active=inactive|sub=dead|result=success|enabled=not-found
|
|
UNIT|name=privoxy.service|load=not-found|active=inactive|sub=dead|result=success|enabled=not-found
|
|
SYSTEMD_STATE_END=1
|
|
BACKUP_AUDIT_FAILURE_EVIDENCE_BEGIN=1
|
|
Sep 23 [IPV6] pve01 homelab-backup-audit[2268991]: TARGET|logical_id=app-filebrowser|kind=APP|state=FRESH|decision=SKIP_FRESH|legacy_age_seconds=14746|generation=app-filebrowser-pve01-20260923T143005Z
|
|
Sep 23 [IPV6] pve01 homelab-backup-audit[2268991]: TARGET|logical_id=app-hbarhub|kind=APP|state=FRESH|decision=SKIP_FRESH|legacy_age_seconds=14574|generation=app-hbarhub-pve01-20260923T143158Z
|
|
Sep 23 [IPV6] pve01 homelab-backup-audit[2268991]: TARGET|logical_id=app-immich|kind=APP|state=FRESH|decision=SKIP_FRESH|legacy_age_seconds=14674|generation=app-immich-pve01-20260923T143041Z
|
|
Sep 23 [IPV6] pve01 homelab-backup-audit[2268991]: TARGET|logical_id=app-skladchik-reports-monitor|kind=APP|state=FRESH|decision=SKIP_FRESH|legacy_age_seconds=14724|generation=app-skladchik-reports-monitor-pve01-20260923T143025Z
|
|
Sep 23 [IPV6] pve01 homelab-backup-audit[2268991]: TARGET|logical_id=control-bootstrap-v2|kind=CONTROL|state=FRESH|age_seconds=36576|generation=control-bootstrap-v2-pve01-20260916T120057Z
|
|
Sep 23 [IPV6] pve01 homelab-backup-audit[2268991]: TARGET|logical_id=vm110|kind=VM|state=FRESH|source=V2|v2_age_seconds=20203|legacy_age_seconds=1688189|replica_count=2|state_id=ct110|generation=v2-ct110-20260923T125546Z-34408ffe4182
|
|
Sep 23 [IPV6] pve01 homelab-backup-audit[2268991]: TARGET|logical_id=vm111|kind=VM|state=FRESH|source=V2|v2_age_seconds=18061|replica_count=2|state_id=ct111|generation=v2-ct111-20260923T133222Z-0d3f5fae9c35
|
|
Sep 23 [IPV6] pve01 homelab-backup-audit[2268991]: TARGET|logical_id=vm112|kind=VM|state=FRESH|source=V2|v2_age_seconds=71147|legacy_age_seconds=1726684|replica_count=2|state_id=ct112|generation=v2-ct112-20260922T224902Z-8457b570052f
|
|
Sep 23 [IPV6] pve01 homelab-backup-audit[2268991]: TARGET|logical_id=vm113|kind=VM|state=FRESH|source=V2|v2_age_seconds=69278|replica_count=2|state_id=ct113|generation=v2-ct113-20260922T231959Z-9fe7d084a330
|
|
Sep 23 [IPV6] pve01 homelab-backup-audit[2268991]: TARGET|logical_id=vm130|kind=VM|state=FRESH|source=V2|v2_age_seconds=61252|replica_count=2|state_id=vm130|generation=v2-vm130-20260923T011558Z-c9498d37d3d7
|
|
Sep 23 [IPV6] pve01 homelab-backup-audit[2268991]: TARGET|logical_id=vm150|kind=VM|state=FRESH|source=V2|v2_age_seconds=66934|legacy_age_seconds=1688305|replica_count=2|state_id=vm150|generation=v2-vm150-20260922T235842Z-71b8fcec59c9
|
|
Sep 23 [IPV6] pve01 homelab-backup-audit[2268991]: TARGET|logical_id=vm160|kind=VM|state=FRESH|source=V2|v2_age_seconds=60152|replica_count=2|state_id=vm160|generation=v2-vm160-20260923T014608Z-f76d7671af5a
|
|
Sep 23 [IPV6] pve01 homelab-backup-audit[2268991]: TARGET|logical_id=vm170|kind=VM|state=FRESH|source=V2|v2_age_seconds=51801|legacy_age_seconds=632926|replica_count=2|state_id=vm170|generation=v2-vm170-20260923T035958Z-923adacbc94b
|
|
Sep 23 [IPV6] pve01 homelab-backup-audit[2268991]: TARGET|logical_id=vm171|kind=VM|state=FRESH|source=V2|v2_age_seconds=58355|legacy_age_seconds=662458|replica_count=2|state_id=vm171|generation=v2-vm171-20260923T021946Z-4d6e4f578075
|
|
Sep 23 [IPV6] pve01 homelab-backup-audit[2268991]: TARGET|logical_id=vm190|kind=VM|state=FRESH|source=V2|v2_age_seconds=54272|replica_count=2|state_id=vm190|generation=v2-vm190-20260923T032759Z-314d9ba2ec35
|
|
Sep 23 [IPV6] pve01 homelab-backup-audit[2268991]: TARGET|logical_id=vm9130|kind=VM|state=FRESH|source=V2|v2_age_seconds=39367|replica_count=2|state_id=vm9130|generation=v2-vm9130-20260923T071932Z-c0492db6535b
|
|
Sep 23 [IPV6] pve01 homelab-backup-audit[2268991]: ISSUE|severity=HIGH|code=LOCK_BUSY|logical_id=/run/homelab-backup-heavy.lock|root_cause=ACTIVE_OR_ORPHAN_LOCK
|
|
Sep 23 [IPV6] pve01 homelab-backup-audit[2268991]: SUMMARY|AUDIT_STATUS=OK|BACKUP_STATUS=RED|ACTIVE_COUNT=16|ISSUE_COUNT=1|CLOUD_A=OK|CLOUD_B=OK|CLOUD_B_DRILL=OK|DR_PROVEN=YES|SOURCE_OFFHOST=OK
|
|
Sep 23 [IPV6] pve01 systemd[1]: homelab-backup-audit.service: Main process exited, code=exited, status=1/FAILURE
|
|
Sep 23 [IPV6] pve01 systemd[1]: homelab-backup-audit.service: Failed with result 'exit-code'.
|
|
Sep 23 [IPV6] pve01 systemd[1]: Failed to start homelab-backup-audit.service - Homelab unified backup and DR audit.
|
|
Sep 23 [IPV6] pve01 systemd[1]: homelab-backup-audit.service: Consumed 2.367s CPU time, 148.3M memory peak.
|
|
Sep 23 [IPV6] pve01 systemd[1]: Starting homelab-backup-audit.service - Homelab unified backup and DR audit...
|
|
Sep 23 [IPV6] pve01 homelab-backup-audit[2372997]: TARGET|logical_id=app-filebrowser|kind=APP|state=FRESH|decision=SKIP_FRESH|legacy_age_seconds=16586|generation=app-filebrowser-pve01-20260923T143005Z
|
|
Sep 23 [IPV6] pve01 homelab-backup-audit[2372997]: TARGET|logical_id=app-hbarhub|kind=APP|state=FRESH|decision=SKIP_FRESH|legacy_age_seconds=16414|generation=app-hbarhub-pve01-20260923T143158Z
|
|
Sep 23 [IPV6] pve01 homelab-backup-audit[2372997]: TARGET|logical_id=app-immich|kind=APP|state=FRESH|decision=SKIP_FRESH|legacy_age_seconds=16514|generation=app-immich-pve01-20260923T143041Z
|
|
Sep 23 [IPV6] pve01 homelab-backup-audit[2372997]: TARGET|logical_id=app-skladchik-reports-monitor|kind=APP|state=FRESH|decision=SKIP_FRESH|legacy_age_seconds=16564|generation=app-skladchik-reports-monitor-pve01-20260923T143025Z
|
|
Sep 23 [IPV6] pve01 homelab-backup-audit[2372997]: TARGET|logical_id=control-bootstrap-v2|kind=CONTROL|state=FRESH|age_seconds=38415|generation=control-bootstrap-v2-pve01-20260916T120057Z
|
|
Sep 23 [IPV6] pve01 homelab-backup-audit[2372997]: TARGET|logical_id=vm110|kind=VM|state=FRESH|source=V2|v2_age_seconds=22042|legacy_age_seconds=1690029|replica_count=2|state_id=ct110|generation=v2-ct110-20260923T125546Z-34408ffe4182
|
|
Sep 23 [IPV6] pve01 homelab-backup-audit[2372997]: TARGET|logical_id=vm111|kind=VM|state=FRESH|source=V2|v2_age_seconds=19900|replica_count=2|state_id=ct111|generation=v2-ct111-20260923T133222Z-0d3f5fae9c35
|
|
Sep 23 [IPV6] pve01 homelab-backup-audit[2372997]: TARGET|logical_id=vm112|kind=VM|state=STALE|source=NONE|v2_age_seconds=72986|legacy_age_seconds=1728524|replica_count=2|state_id=ct112|generation=vm112-pve01-20260903T185707Z
|
|
Sep 23 [IPV6] pve01 homelab-backup-audit[2372997]: TARGET|logical_id=vm113|kind=VM|state=FRESH|source=V2|v2_age_seconds=71117|replica_count=2|state_id=ct113|generation=v2-ct113-20260922T231959Z-9fe7d084a330
|
|
Sep 23 [IPV6] pve01 homelab-backup-audit[2372997]: TARGET|logical_id=vm130|kind=VM|state=FRESH|source=V2|v2_age_seconds=63091|replica_count=2|state_id=vm130|generation=v2-vm130-20260923T011558Z-c9498d37d3d7
|
|
Sep 23 [IPV6] pve01 homelab-backup-audit[2372997]: TARGET|logical_id=vm150|kind=VM|state=FRESH|source=V2|v2_age_seconds=68773|legacy_age_seconds=1690145|replica_count=2|state_id=vm150|generation=v2-vm150-20260922T235842Z-71b8fcec59c9
|
|
Sep 23 [IPV6] pve01 homelab-backup-audit[2372997]: TARGET|logical_id=vm160|kind=VM|state=FRESH|source=V2|v2_age_seconds=61991|replica_count=2|state_id=vm160|generation=v2-vm160-20260923T014608Z-f76d7671af5a
|
|
Sep 23 [IPV6] pve01 homelab-backup-audit[2372997]: TARGET|logical_id=vm170|kind=VM|state=FRESH|source=V2|v2_age_seconds=53640|legacy_age_seconds=634766|replica_count=2|state_id=vm170|generation=v2-vm170-20260923T035958Z-923adacbc94b
|
|
Sep 23 [IPV6] pve01 homelab-backup-audit[2372997]: TARGET|logical_id=vm171|kind=VM|state=FRESH|source=V2|v2_age_seconds=60194|legacy_age_seconds=664298|replica_count=2|state_id=vm171|generation=v2-vm171-20260923T021946Z-4d6e4f578075
|
|
Sep 23 [IPV6] pve01 homelab-backup-audit[2372997]: TARGET|logical_id=vm190|kind=VM|state=FRESH|source=V2|v2_age_seconds=56111|replica_count=2|state_id=vm190|generation=v2-vm190-20260923T032759Z-314d9ba2ec35
|
|
Sep 23 [IPV6] pve01 homelab-backup-audit[2372997]: TARGET|logical_id=vm9130|kind=VM|state=FRESH|source=V2|v2_age_seconds=41206|replica_count=2|state_id=vm9130|generation=v2-vm9130-20260923T071932Z-c0492db6535b
|
|
Sep 23 [IPV6] pve01 homelab-backup-audit[2372997]: ISSUE|severity=HIGH|code=RPO_BREACH|logical_id=vm112|root_cause=NO_FRESH_V2_OR_SEALED_QUORUM
|
|
Sep 23 [IPV6] pve01 homelab-backup-audit[2372997]: ISSUE|severity=HIGH|code=LOCK_BUSY|logical_id=/run/homelab-backup-heavy.lock|root_cause=ACTIVE_OR_ORPHAN_LOCK
|
|
Sep 23 [IPV6] pve01 homelab-backup-audit[2372997]: SUMMARY|AUDIT_STATUS=OK|BACKUP_STATUS=RED|ACTIVE_COUNT=16|ISSUE_COUNT=2|CLOUD_A=OK|CLOUD_B=OK|CLOUD_B_DRILL=OK|DR_PROVEN=YES|SOURCE_OFFHOST=OK
|
|
Sep 23 [IPV6] pve01 systemd[1]: homelab-backup-audit.service: Main process exited, code=exited, status=1/FAILURE
|
|
Sep 23 [IPV6] pve01 systemd[1]: homelab-backup-audit.service: Failed with result 'exit-code'.
|
|
Sep 23 [IPV6] pve01 systemd[1]: Failed to start homelab-backup-audit.service - Homelab unified backup and DR audit.
|
|
Sep 23 [IPV6] pve01 systemd[1]: homelab-backup-audit.service: Consumed 2.438s CPU time, 148.4M memory peak.
|
|
Sep 23 [IPV6] pve01 systemd[1]: Starting homelab-backup-audit.service - Homelab unified backup and DR audit...
|
|
Sep 23 [IPV6] pve01 homelab-backup-audit[2466119]: TARGET|logical_id=app-filebrowser|kind=APP|state=FRESH|decision=SKIP_FRESH|legacy_age_seconds=18320|generation=app-filebrowser-pve01-20260923T143005Z
|
|
Sep 23 [IPV6] pve01 homelab-backup-audit[2466119]: TARGET|logical_id=app-hbarhub|kind=APP|state=FRESH|decision=SKIP_FRESH|legacy_age_seconds=18148|generation=app-hbarhub-pve01-20260923T143158Z
|
|
Sep 23 [IPV6] pve01 homelab-backup-audit[2466119]: TARGET|logical_id=app-immich|kind=APP|state=FRESH|decision=SKIP_FRESH|legacy_age_seconds=18248|generation=app-immich-pve01-20260923T143041Z
|
|
Sep 23 [IPV6] pve01 homelab-backup-audit[2466119]: TARGET|logical_id=app-skladchik-reports-monitor|kind=APP|state=FRESH|decision=SKIP_FRESH|legacy_age_seconds=18298|generation=app-skladchik-reports-monitor-pve01-20260923T143025Z
|
|
Sep 23 [IPV6] pve01 homelab-backup-audit[2466119]: TARGET|logical_id=control-bootstrap-v2|kind=CONTROL|state=FRESH|age_seconds=40150|generation=control-bootstrap-v2-pve01-20260916T120057Z
|
|
Sep 23 [IPV6] pve01 homelab-backup-audit[2466119]: TARGET|logical_id=vm110|kind=VM|state=FRESH|source=V2|v2_age_seconds=23777|legacy_age_seconds=1691763|replica_count=2|state_id=ct110|generation=v2-ct110-20260923T125546Z-34408ffe4182
|
|
Sep 23 [IPV6] pve01 homelab-backup-audit[2466119]: TARGET|logical_id=vm111|kind=VM|state=FRESH|source=V2|v2_age_seconds=21635|replica_count=2|state_id=ct111|generation=v2-ct111-20260923T133222Z-0d3f5fae9c35
|
|
Sep 23 [IPV6] pve01 homelab-backup-audit[2466119]: TARGET|logical_id=vm112|kind=VM|state=STALE|source=NONE|v2_age_seconds=74721|legacy_age_seconds=1730258|replica_count=2|state_id=ct112|generation=vm112-pve01-20260903T185707Z
|
|
Sep 23 [IPV6] pve01 homelab-backup-audit[2466119]: TARGET|logical_id=vm113|kind=VM|state=STALE|source=NONE|v2_age_seconds=72852|replica_count=2|state_id=ct113
|
|
Sep 23 [IPV6] pve01 homelab-backup-audit[2466119]: TARGET|logical_id=vm130|kind=VM|state=FRESH|source=V2|v2_age_seconds=64826|replica_count=2|state_id=vm130|generation=v2-vm130-20260923T011558Z-c9498d37d3d7
|
|
Sep 23 [IPV6] pve01 homelab-backup-audit[2466119]: TARGET|logical_id=vm150|kind=VM|state=FRESH|source=V2|v2_age_seconds=70508|legacy_age_seconds=1691879|replica_count=2|state_id=vm150|generation=v2-vm150-20260922T235842Z-71b8fcec59c9
|
|
Sep 23 [IPV6] pve01 homelab-backup-audit[2466119]: TARGET|logical_id=vm160|kind=VM|state=FRESH|source=V2|v2_age_seconds=63726|replica_count=2|state_id=vm160|generation=v2-vm160-20260923T014608Z-f76d7671af5a
|
|
Sep 23 [IPV6] pve01 homelab-backup-audit[2466119]: TARGET|logical_id=vm170|kind=VM|state=FRESH|source=V2|v2_age_seconds=55375|legacy_age_seconds=636500|replica_count=2|state_id=vm170|generation=v2-vm170-20260923T035958Z-923adacbc94b
|
|
Sep 23 [IPV6] pve01 homelab-backup-audit[2466119]: TARGET|logical_id=vm171|kind=VM|state=FRESH|source=V2|v2_age_seconds=61929|legacy_age_seconds=666032|replica_count=2|state_id=vm171|generation=v2-vm171-20260923T021946Z-4d6e4f578075
|
|
Sep 23 [IPV6] pve01 homelab-backup-audit[2466119]: TARGET|logical_id=vm190|kind=VM|state=FRESH|source=V2|v2_age_seconds=57846|replica_count=2|state_id=vm190|generation=v2-vm190-20260923T032759Z-314d9ba2ec35
|
|
Sep 23 [IPV6] pve01 homelab-backup-audit[2466119]: TARGET|logical_id=vm9130|kind=VM|state=FRESH|source=V2|v2_age_seconds=42941|replica_count=2|state_id=vm9130|generation=v2-vm9130-20260923T071932Z-c0492db6535b
|
|
Sep 23 [IPV6] pve01 homelab-backup-audit[2466119]: ISSUE|severity=HIGH|code=RPO_BREACH|logical_id=vm112|root_cause=NO_FRESH_V2_OR_SEALED_QUORUM
|
|
Sep 23 [IPV6] pve01 homelab-backup-audit[2466119]: ISSUE|severity=HIGH|code=RPO_BREACH|logical_id=vm113|root_cause=NO_FRESH_V2_OR_SEALED_QUORUM
|
|
Sep 23 [IPV6] pve01 homelab-backup-audit[2466119]: SUMMARY|AUDIT_STATUS=OK|BACKUP_STATUS=RED|ACTIVE_COUNT=16|ISSUE_COUNT=2|CLOUD_A=OK|CLOUD_B=OK|CLOUD_B_DRILL=OK|DR_PROVEN=YES|SOURCE_OFFHOST=OK
|
|
Sep 23 [IPV6] pve01 systemd[1]: homelab-backup-audit.service: Main process exited, code=exited, status=1/FAILURE
|
|
Sep 23 [IPV6] pve01 systemd[1]: homelab-backup-audit.service: Failed with result 'exit-code'.
|
|
Sep 23 [IPV6] pve01 systemd[1]: Failed to start homelab-backup-audit.service - Homelab unified backup and DR audit.
|
|
Sep 23 [IPV6] pve01 systemd[1]: homelab-backup-audit.service: Consumed 2.421s CPU time, 148M memory peak.
|
|
Sep 23 [IPV6] pve01 systemd[1]: Starting homelab-backup-audit.service - Homelab unified backup and DR audit...
|
|
Sep 23 [IPV6] pve01 homelab-backup-audit[2565382]: TARGET|logical_id=app-filebrowser|kind=APP|state=FRESH|decision=SKIP_FRESH|legacy_age_seconds=20177|generation=app-filebrowser-pve01-20260923T143005Z
|
|
Sep 23 [IPV6] pve01 homelab-backup-audit[2565382]: TARGET|logical_id=app-hbarhub|kind=APP|state=FRESH|decision=SKIP_FRESH|legacy_age_seconds=20005|generation=app-hbarhub-pve01-20260923T143158Z
|
|
Sep 23 [IPV6] pve01 homelab-backup-audit[2565382]: TARGET|logical_id=app-immich|kind=APP|state=FRESH|decision=SKIP_FRESH|legacy_age_seconds=20105|generation=app-immich-pve01-20260923T143041Z
|
|
Sep 23 [IPV6] pve01 homelab-backup-audit[2565382]: TARGET|logical_id=app-skladchik-reports-monitor|kind=APP|state=FRESH|decision=SKIP_FRESH|legacy_age_seconds=20155|generation=app-skladchik-reports-monitor-pve01-20260923T143025Z
|
|
Sep 23 [IPV6] pve01 homelab-backup-audit[2565382]: TARGET|logical_id=control-bootstrap-v2|kind=CONTROL|state=FRESH|age_seconds=42006|generation=control-bootstrap-v2-pve01-20260916T120057Z
|
|
Sep 23 [IPV6] pve01 homelab-backup-audit[2565382]: TARGET|logical_id=vm110|kind=VM|state=FRESH|source=V2|v2_age_seconds=25633|legacy_age_seconds=1693620|replica_count=2|state_id=ct110|generation=v2-ct110-20260923T125546Z-34408ffe4182
|
|
Sep 23 [IPV6] pve01 homelab-backup-audit[2565382]: TARGET|logical_id=vm111|kind=VM|state=FRESH|source=V2|v2_age_seconds=23491|replica_count=2|state_id=ct111|generation=v2-ct111-20260923T133222Z-0d3f5fae9c35
|
|
Sep 23 [IPV6] pve01 homelab-backup-audit[2565382]: TARGET|logical_id=vm112|kind=VM|state=STALE|source=NONE|v2_age_seconds=76577|legacy_age_seconds=1732115|replica_count=2|state_id=ct112|generation=vm112-pve01-20260903T185707Z
|
|
Sep 23 [IPV6] pve01 homelab-backup-audit[2565382]: TARGET|logical_id=vm113|kind=VM|state=STALE|source=NONE|v2_age_seconds=74708|replica_count=2|state_id=ct113
|
|
Sep 23 [IPV6] pve01 homelab-backup-audit[2565382]: TARGET|logical_id=vm130|kind=VM|state=FRESH|source=V2|v2_age_seconds=66682|replica_count=2|state_id=vm130|generation=v2-vm130-20260923T011558Z-c9498d37d3d7
|
|
Sep 23 [IPV6] pve01 homelab-backup-audit[2565382]: TARGET|logical_id=vm150|kind=VM|state=STALE|source=NONE|v2_age_seconds=72364|legacy_age_seconds=1693736|replica_count=2|state_id=vm150|generation=vm150-pve01-20260904T053517Z
|
|
Sep 23 [IPV6] pve01 homelab-backup-audit[2565382]: TARGET|logical_id=vm160|kind=VM|state=FRESH|source=V2|v2_age_seconds=65582|replica_count=2|state_id=vm160|generation=v2-vm160-20260923T014608Z-f76d7671af5a
|
|
Sep 23 [IPV6] pve01 homelab-backup-audit[2565382]: TARGET|logical_id=vm170|kind=VM|state=FRESH|source=V2|v2_age_seconds=57231|legacy_age_seconds=638357|replica_count=2|state_id=vm170|generation=v2-vm170-20260923T035958Z-923adacbc94b
|
|
Sep 23 [IPV6] pve01 homelab-backup-audit[2565382]: TARGET|logical_id=vm171|kind=VM|state=FRESH|source=V2|v2_age_seconds=63785|legacy_age_seconds=667889|replica_count=2|state_id=vm171|generation=v2-vm171-20260923T021946Z-4d6e4f578075
|
|
Sep 23 [IPV6] pve01 homelab-backup-audit[2565382]: TARGET|logical_id=vm190|kind=VM|state=FRESH|source=V2|v2_age_seconds=59702|replica_count=2|state_id=vm190|generation=v2-vm190-20260923T032759Z-314d9ba2ec35
|
|
Sep 23 [IPV6] pve01 homelab-backup-audit[2565382]: TARGET|logical_id=vm9130|kind=VM|state=FRESH|source=V2|v2_age_seconds=44797|replica_count=2|state_id=vm9130|generation=v2-vm9130-20260923T071932Z-c0492db6535b
|
|
Sep 23 [IPV6] pve01 homelab-backup-audit[2565382]: ISSUE|severity=HIGH|code=RPO_BREACH|logical_id=vm112|root_cause=NO_FRESH_V2_OR_SEALED_QUORUM
|
|
Sep 23 [IPV6] pve01 homelab-backup-audit[2565382]: ISSUE|severity=HIGH|code=RPO_BREACH|logical_id=vm113|root_cause=NO_FRESH_V2_OR_SEALED_QUORUM
|
|
Sep 23 [IPV6] pve01 homelab-backup-audit[2565382]: ISSUE|severity=HIGH|code=RPO_BREACH|logical_id=vm150|root_cause=NO_FRESH_V2_OR_SEALED_QUORUM
|
|
BACKUP_AUDIT_FAILURE_EVIDENCE_END=1
|
|
HEALTH_FILES_BEGIN=1
|
|
HEALTH_FILE=cluster-admin-vm-mail-cloud-backup.txt|mtime=2026-07-09 08:07:47.213452046 +0300|sha256=e03b236f41f8d66f070ecd3234b3ff0836a44d972fd6f6abc721b9683fa22053
|
|
STATUS=OK TS=2026-07-09T05:07:47Z TYPE=cluster-admin-vm-mail-cloud-backup BAD=0 VMID=180 NODE=pve02 MODE=MAIL_CLOUD_2COPY_VERIFY_OK LOCAL_STATUS=OK REPORT=/var/lib/homelab-cluster-admin-vm-mail-cloud-backup/latest.md
|
|
HEALTH_FILE=alertmanager-backup.txt|mtime=2026-07-09 10:50:17.826712693 +0300|sha256=f5a8a0ad29606ebbdcdcb52a997e9ee07ddec9ab7eab6bb0932c3f4e14eba97a
|
|
STATUS=OK TS=2026-07-09T07:50:16Z TYPE=alertmanager-backup JOB=homelab-pve01-app-alertmanager-20260709T075016Z SOURCE=alertmanager_config SIZE=1842 SHA256=58fb1b3467a962acff44bf75cb274e3a17c06afdbd3a25314a904f75c8048728 LOCAL_BACKUP_DELETED=true NOTE=service-readiness-backup-proof
|
|
HEALTH_FILE=mealie-backup.txt|mtime=2026-07-09 19:17:35.017044807 +0300|sha256=8043626489c0b50dc30f11771e15940c0cb27c99cebdf482cf7dfba83571d074
|
|
STATUS=OK TS=2026-07-09T16:17:31Z TYPE=mealie-backup JOB=homelab-pve01-app-mealie-20260709T051713Z SOURCE=latest-appbackup SIZE=912487 LOCAL_BACKUP_DELETED=true JSON=/var/lib/homelab-backup/apps/latest-mealie.json NOTE=service-readiness-backup-proof
|
|
HEALTH_FILE=karakeep-backup.txt|mtime=2026-07-09 22:28:35.572436986 +0300|sha256=79a25462c21ee9006462c6dc6fb7c2b21a4e7c297f585b770ed1e9a77e718230
|
|
STATUS=OK TS=2026-07-09T19:28:34Z TYPE=karakeep-backup BAD=0 APP=karakeep JOB=homelab-pve01-app-karakeep-20260709T051713Z SIZE=23140 LOCAL_BACKUP_DELETED=true JSON=/var/lib/homelab-backup/apps/latest-karakeep.json
|
|
HEALTH_FILE=backup-restore-dashboard.txt|mtime=2026-07-03 00:45:30.941379161 +0300|sha256=51b95b7465dc040ff080c8a8341b23c9c0fdd90cfe0675ccd3f705b55590d815
|
|
STATUS=OK
|
|
TS=20260702T214530Z
|
|
TYPE=backup-restore-dashboard-rebuilt
|
|
ITEMS=13
|
|
NOT_OK=0
|
|
NOT_OK_LIST=none
|
|
ITEM name=backup.txt exists=true ok=true
|
|
ITEM name=restore-drill-index.txt exists=true ok=true
|
|
ITEM name=paperless-backup.txt exists=true ok=true
|
|
ITEM name=paperless-offhost.txt exists=true ok=true
|
|
ITEM name=paperless-restore.txt exists=true ok=true
|
|
ITEM name=npmplus-kuma-config-backup.txt exists=true ok=true
|
|
ITEM name=npmplus-kuma-config-offhost.txt exists=true ok=true
|
|
ITEM name=npmplus-kuma-config-restore.txt exists=true ok=true
|
|
ITEM name=netbird-peers.txt exists=true ok=true
|
|
ITEM name=crowdsec-capi-netbird-final.txt exists=true ok=true
|
|
ITEM name=adguard-rewrite-sync.txt exists=true ok=true
|
|
ITEM name=router-running-config-mail-cloud.txt exists=true ok=true
|
|
ITEM name=mail-cloud-restore-drill.txt exists=true ok=true
|
|
HEALTH_FILE=backup-matrix-cluster-final-seal.txt|mtime=2026-07-13 20:51:32.422338540 +0300|sha256=dcd06c2ff207c2922fc68d4df4b42ffa4b6680785e532783772e764feb828aa1
|
|
STATUS=OK TS=2026-07-13T17:51:31Z TYPE=backup-matrix-cluster-final-seal BAD=0 CURRENT_STAGE_COMPLETE=YES MATRIX_STATUS=OK JOBS_TOTAL=35 JOBS_OK=31 JOBS_BAD=0 JOBS_UNMAPPED=0 JOBS_DISABLED=4 READ_WRITE_OVERLAP=0 VM_BACKUP_HEALTH=OK VM_FULL_BACKUP=OK EDGE_VM_CANONICAL_BACKUP=homelab-edge-vm-offhost-freshness.timer OLD_EDGE_TIMER_REENABLED=NO CLUSTER_PASSPORT=OK FINAL_READINESS=OK FINAL_READINESS_PASS=53 FINAL_READINESS_FAIL=0 OVERALL=OK CLUSTER_ADMIN=OK PROBE_OK=4 PROBE_BAD=0 WEBPANEL=OK FAILED_UNITS=0 CLUSTER_NODES=3 QUORATE=YES GENERATED_SYNC_HEAD=597119391e7b265b85fd9a76644e4e33af47e469 DESIRED_STATE_DIRTY=0 MATRIX_SHA256=7613bcb25d32e61fac114da13af81e38354ec22a012a6deeaa168ee4cc454f87 MATRIX_STATE_SHA256=14a31112aac5d577ae01df567921b093b2fedcf7189c4cc8f009216ee0749380 CLUSTER_PASSPORT_SHA256=1b8b924ae0e5ce6872fe5671318094ce494806c2db7efe28299b5ff49adc0d0d FINAL_READINESS_SHA256=ac5159fa8c497afe021b5cb232f08cfec0e1a4d516d8aae966ba03f7034f1412 OVERALL_SHA256=6ae529bfdf5811f686d3b53978f453ce4f8757c74fc1be105de219b13a61ba46 PRESEAL_SHA256=7ea1f710cd311bd51dba05a313b4c540f57d97407f008881e502fc213b4b3c99 EVIDENCE_MANIFEST_SHA256=b4c68312026e1328b208a74090891f4809ab31ec69069e94a3d6902429bf3566 BACKUP_RUN=NO RESTORE_RUN=NO CLOUD_WRITE_RUN=NO RETENTION_RUN=NO NEXT_ACTION=AUDIT_AND_MIGRATE_SPECIAL_VM_EMERGENCY_AND_NON_APP_BACKUP_FAMILIES EVIDENCE_ROOT=/var/lib/homelab-change-sets/backup-matrix-cluster-final-seal/20260713T175131Z
|
|
HEALTH_FILE=actual-backup.txt|mtime=2026-07-09 15:53:45.248005185 +0300|sha256=72d1b31e22e22ac91af66e3d8ae46b20b3b9297809ace524eece22031458fb05
|
|
STATUS=OK TS=2026-07-09T12:53:43Z TYPE=actual-budget-backup JOB=homelab-pve01-app-actual-budget-20260709T051713Z SOURCE=latest-appbackup SIZE=2011 LOCAL_BACKUP_DELETED=true JSON=/var/lib/homelab-backup/apps/latest-actual-budget.json NOTE=service-readiness-backup-proof
|
|
HEALTH_FILE=authentik-backup.txt|mtime=2026-08-30 21:08:08.459821635 +0300|sha256=8888712fdc8c5fd18ef0b691eb5d379fbd71af8f5dbda4e917b5a1493d12c046
|
|
STATUS=FAIL
|
|
TS=2026-08-30T18:08:08Z
|
|
TYPE=backup-matrix-generated
|
|
JOB_ID=homelab_appbackup_postgres_batch
|
|
JOB_NAME=homelab-appbackup-postgres-batch
|
|
MATRIX_MODE=authoritative
|
|
TIMER=homelab-appbackup-postgres-batch.timer
|
|
SERVICE=homelab-appbackup-postgres-batch.service
|
|
TIMER_ENABLED=not-found
|
|
TIMER_ACTIVE=inactive
|
|
SOURCE_KIND=systemd
|
|
EVIDENCE_LEVEL=execution_only
|
|
SERVICE_RESULT=success
|
|
SERVICE_EXEC_MAIN_STATUS=0
|
|
SOURCE_HEALTH=systemd:homelab-appbackup-postgres-batch.service
|
|
SOURCE_STATUS=OK
|
|
SOURCE_MTIME_UTC=2026-07-24T02:18:04Z
|
|
AGE_SECONDS=3253804
|
|
MAX_AGE_SECONDS=172800
|
|
FRESHNESS=STALE
|
|
LEGACY_SOURCE_PATH=/var/lib/homelab-health/authentik-backup.txt
|
|
CANONICAL_JOB_ID=homelab_appbackup_postgres_batch
|
|
CANONICAL_STATE=/var/lib/homelab-backup-matrix/state.json
|
|
HEALTH_FILE=cluster-admin-stage3-postapply-backup.txt|mtime=2026-07-13 06:44:42.760995774 +0300|sha256=c00feef82c76522380776c84bf5bdb742ca2105c0dc2eeb1b596db2880facef9
|
|
STATUS=OK TS=2026-07-13T03:43:59Z TYPE=cluster-admin-stage3-postapply-backup BAD=0 COMMAND_PHYSICAL_LINES=1 PRODUCTION_CHANGED=POSTAPPLY_BACKUP_AND_CONTROLLED_COLLECTOR_RUN PRODUCTION_DATABASE_CHANGED=COLLECTOR_NORMAL_OPERATION_ONLY PRODUCTION_SCHEMA_CHANGED=NO LIVE_APPLICATION_CODE_CHANGED=NO DESIRED_STATE_CHANGED=NO SERVICE_RESTARTED=NO CLOUD_WRITE_RUN=YES_CONTROLLED_POSTAPPLY_BACKUP LIVE_APP_SHA256=a330a2d33ee5a8a57778e3541179e3faebcc0531600a504df744435e80a93b34 DESIRED_STATE_HEAD=b974057b536ece861299d0b230c06cc0e6702c39 APPBACKUP=VERIFIED APPBACKUP_JOB=homelab-pve01-app-cluster-admin-incident-engine-20260713T034400Z APPBACKUP_SIZE_BYTES=1791603 APPBACKUP_RESTORE_CHECK=OK APPBACKUP_ROUTING=VERIFIED COLLECTOR_CONTROL_RUN=OK COLLECTOR_RUN_ID=3113 CANONICAL_CONSISTENCY=0|0|0|0|0 INCIDENT_ENGINE=ACTIVE COLLECTOR_TIMER=ACTIVE VM180_FAILED_UNITS=0 READY_FOR_STAGE3_FINAL_SEAL=YES STAGE3_CLOSED=NO POSTAPPLY_ROOT=/var/lib/homelab-stage3-postapply/20260713T034359Z
|
|
HEALTH_FILE=netbird-vps-trust.txt|mtime=2026-07-18 01:54:49.485000918 +0300|sha256=643c858c4b1cfa77a7914349ccb05434a94e27a358320b687658684665d5c1ac
|
|
STATUS=OK
|
|
TS=2026-07-17T04:52:15Z
|
|
TYPE=backup-matrix-generated
|
|
JOB_ID=netbird_vps_backup
|
|
JOB_NAME=netbird-vps-backup
|
|
MATRIX_MODE=authoritative
|
|
TIMER=netbird-vps-backup.timer
|
|
SERVICE=netbird-vps-backup.service
|
|
TIMER_ENABLED=enabled
|
|
TIMER_ACTIVE=active
|
|
SOURCE_KIND=health_file
|
|
EVIDENCE_LEVEL=artifact_or_health
|
|
SERVICE_RESULT=success
|
|
SERVICE_EXEC_MAIN_STATUS=0
|
|
SOURCE_HEALTH=/var/lib/homelab-backup-matrix/evidence/netbird_vps_backup/0-netbird-vps-trust.txt
|
|
SOURCE_STATUS=OK
|
|
SOURCE_MTIME_UTC=2026-07-07T15:24:59Z
|
|
AGE_SECONDS=826036
|
|
MAX_AGE_SECONDS=864000
|
|
FRESHNESS=FRESH
|
|
LEGACY_SOURCE_PATH=/var/lib/homelab-health/netbird-vps-trust.txt
|
|
CANONICAL_JOB_ID=netbird_vps_backup
|
|
CANONICAL_STATE=/var/lib/homelab-backup-matrix/state.json
|
|
HEALTH_FILE=extended-appbackup.txt|mtime=2026-08-30 21:08:08.661414778 +0300|sha256=00167921437640beadbd7c5cc7323845b3893f85c161781d0899da9fba63f991
|
|
STATUS=FAIL
|
|
TS=2026-08-30T18:08:08Z
|
|
TYPE=backup-matrix-generated
|
|
JOB_ID=homelab_extended_appbackup
|
|
JOB_NAME=homelab-extended-appbackup
|
|
MATRIX_MODE=authoritative
|
|
TIMER=homelab-extended-appbackup.timer
|
|
SERVICE=homelab-extended-appbackup.service
|
|
TIMER_ENABLED=not-found
|
|
TIMER_ACTIVE=inactive
|
|
SOURCE_KIND=health_file
|
|
EVIDENCE_LEVEL=artifact_or_health
|
|
SERVICE_RESULT=success
|
|
SERVICE_EXEC_MAIN_STATUS=0
|
|
SOURCE_HEALTH=/var/lib/homelab-backup-matrix/evidence/homelab_extended_appbackup/0-extended-appbackup.txt
|
|
SOURCE_STATUS=OK
|
|
SOURCE_MTIME_UTC=2026-08-06T21:34:02Z
|
|
AGE_SECONDS=2061246
|
|
MAX_AGE_SECONDS=172800
|
|
FRESHNESS=STALE
|
|
LEGACY_SOURCE_PATH=/var/lib/homelab-health/extended-appbackup.txt
|
|
CANONICAL_JOB_ID=homelab_extended_appbackup
|
|
CANONICAL_STATE=/var/lib/homelab-backup-matrix/state.json
|
|
HEALTH_FILE=healthchecks-backup.txt|mtime=2026-07-09 11:51:31.108386138 +0300|sha256=f2012e88374f88b98e419adb29ee39db9dd58540ed8f20d7894813eabad36301
|
|
STATUS=OK TS=2026-07-09T08:51:29Z TYPE=healthchecks-backup JOB=homelab-pve01-app-healthchecks-20260709T051713Z SOURCE=latest-appbackup SIZE=687 LOCAL_BACKUP_DELETED=true JSON=/var/lib/homelab-backup/apps/latest-healthchecks.json NOTE=service-readiness-backup-proof
|
|
HEALTH_FILE=backup-scheduler-rpo-hardening.json|mtime=2026-08-10 13:51:49.375986623 +0300|sha256=e73927d669e1d066d03cd711449f9533ce018da041a86432c8550624a5a6c55f
|
|
{
|
|
"status": "OK",
|
|
"root_cause_count": 0,
|
|
"production_changed": true,
|
|
"cloud_changed": false,
|
|
"backup_started": false,
|
|
"applied_at_utc": "2026-08-10T10:51:49Z",
|
|
"target_bytes": true,
|
|
"persistent_stamps_rebased": true,
|
|
"timers_started": true,
|
|
"no_immediate_service_start": true,
|
|
"next": {
|
|
"vm": "Mon 2026-08-10 [IPV6] MSK",
|
|
"app": "Mon 2026-08-10 [IPV6] MSK"
|
|
}
|
|
}
|
|
HEALTH_FILE=minio-backup.txt|mtime=2026-07-09 15:13:16.876111706 +0300|sha256=e4965b27706d848070d5ca6c2e974a14c520465f3b1575fda1ab671d6cf0e124
|
|
STATUS=OK TS=2026-07-09T12:13:14Z TYPE=minio-backup JOB=homelab-pve01-app-minio-20260709T051713Z SOURCE=latest-appbackup SIZE=4585 LOCAL_BACKUP_DELETED=true JSON=/var/lib/homelab-backup/apps/latest-minio.json NOTE=service-readiness-backup-proof
|
|
HEALTH_FILE=backup-framework.txt|mtime=2026-08-30 21:08:08.850863900 +0300|sha256=192caeb6ecfd0d386aa32fcf838229854ea73f3abb7d46343cd8f1b4acf7b2ab
|
|
STATUS=FAIL
|
|
TS=2026-08-30T18:08:08Z
|
|
TYPE=backup-matrix-generated
|
|
JOB_ID=homelab_verified_backup_health
|
|
JOB_NAME=homelab-verified-backup-health
|
|
MATRIX_MODE=authoritative
|
|
TIMER=homelab-verified-backup-health.timer
|
|
SERVICE=homelab-verified-backup-health.service
|
|
TIMER_ENABLED=not-found
|
|
TIMER_ACTIVE=inactive
|
|
SOURCE_KIND=health_file
|
|
EVIDENCE_LEVEL=artifact_or_health
|
|
SERVICE_RESULT=success
|
|
SERVICE_EXEC_MAIN_STATUS=0
|
|
SOURCE_HEALTH=/var/lib/homelab-backup-matrix/evidence/homelab_verified_backup_health/1-latest-emergency-pack.json
|
|
SOURCE_STATUS=OK
|
|
SOURCE_MTIME_UTC=2026-08-06T20:59:56Z
|
|
AGE_SECONDS=2063292
|
|
MAX_AGE_SECONDS=172800
|
|
FRESHNESS=STALE
|
|
LEGACY_SOURCE_PATH=/var/lib/homelab-health/backup-framework.txt
|
|
CANONICAL_JOB_ID=homelab_verified_backup_health
|
|
CANONICAL_STATE=/var/lib/homelab-backup-matrix/state.json
|
|
HEALTH_FILE=audiobookshelf-backup.txt|mtime=2026-07-09 21:22:25.670486930 +0300|sha256=fb81212dd08251ede49f921b0fb61b86a8f4446af80bdd42172acbbaf60b8e32
|
|
STATUS=OK TS=2026-07-09T18:22:24Z TYPE=audiobookshelf-backup BAD=0 APP=audiobookshelf JOB=homelab-pve01-app-audiobookshelf-20260709T051713Z SIZE=23094 LOCAL_BACKUP_DELETED=true JSON=/var/lib/homelab-backup/apps/latest-audiobookshelf.json
|
|
HEALTH_FILE=filebrowser-backup.txt|mtime=2026-08-30 21:08:08.363475316 +0300|sha256=e8d06faa33f0c464b6074e5c121ec8aea9406d532097400a339764bcc71b009c
|
|
STATUS=FAIL
|
|
TS=2026-08-30T18:08:08Z
|
|
TYPE=backup-matrix-generated
|
|
JOB_ID=filebrowser_backup
|
|
JOB_NAME=filebrowser-backup
|
|
MATRIX_MODE=authoritative
|
|
TIMER=filebrowser-backup.timer
|
|
SERVICE=filebrowser-backup.service
|
|
TIMER_ENABLED=not-found
|
|
TIMER_ACTIVE=inactive
|
|
SOURCE_KIND=health_file
|
|
EVIDENCE_LEVEL=artifact_or_health
|
|
SERVICE_RESULT=success
|
|
SERVICE_EXEC_MAIN_STATUS=0
|
|
SOURCE_HEALTH=/var/lib/homelab-backup/apps/latest-filebrowser.json
|
|
SOURCE_STATUS=OK
|
|
SOURCE_MTIME_UTC=2026-08-06T21:28:05Z
|
|
AGE_SECONDS=2061603
|
|
MAX_AGE_SECONDS=259200
|
|
FRESHNESS=STALE
|
|
LEGACY_SOURCE_PATH=/var/lib/homelab-health/filebrowser-backup.txt
|
|
CANONICAL_JOB_ID=filebrowser_backup
|
|
CANONICAL_STATE=/var/lib/homelab-backup-matrix/state.json
|
|
HEALTH_FILE=netbox-backup.txt|mtime=2026-07-09 13:23:55.693530989 +0300|sha256=0cf3e80b9d64332d56d5d6bb6fd0b09e3de366a6817b7cf4ce2fcca80f5ccd0c
|
|
STATUS=OK TS=2026-07-09T10:23:50Z TYPE=netbox-backup JOB=homelab-pve01-app-netbox-20260709T072101Z SOURCE=latest-appbackup SIZE=25860637 LOCAL_BACKUP_DELETED=true JSON=/var/lib/homelab-backup/apps/latest-netbox.json NOTE=service-readiness-backup-proof
|
|
HEALTH_FILE=linkding-backup.txt|mtime=2026-07-09 20:00:19.838194359 +0300|sha256=cb3766255eec88a3a0a70447fb507ced5065345636fce5c4d760012f7e7944ab
|
|
STATUS=OK TS=2026-07-09T17:00:18Z TYPE=linkding-backup BAD=0 APP=linkding JOB=homelab-pve01-app-linkding-20260709T051713Z SIZE=40380 LOCAL_BACKUP_DELETED=true JSON=/var/lib/homelab-backup/apps/latest-linkding.json
|
|
HEALTH_FILE=n8n-backup.txt|mtime=2026-07-09 22:39:24.237551932 +0300|sha256=723af1f45fc9e9e68e95cccc9747fb25b6bfef068c71060de1aaf260c0534774
|
|
STATUS=OK TS=2026-07-09T19:39:23Z TYPE=n8n-backup BAD=0 APP=n8n JOB=homelab-pve01-app-n8n-20260709T051713Z SIZE=92667 LOCAL_BACKUP_DELETED=true JSON=/var/lib/homelab-backup/apps/latest-n8n.json
|
|
HEALTH_FILE=crypto-portfolio-backup.txt|mtime=2026-07-13 11:12:13.302397603 +0300|sha256=cf13d8b15a64fbb2fa412e32ce4eafdc732b045fe82e789928e768ace92361eb
|
|
STATUS=OK TS=2026-07-13T08:12:13Z TYPE=crypto-portfolio-backup APP=crypto-portfolio JOB=homelab-pve01-app-crypto-portfolio-20260713T075801Z PATH_REL=hb/a/cp-260713-075801 ARCHIVE_SIZE_BYTES=137496 FILE_COUNT=19 ARCHIVE_SHA256=d6d3f6a397c5e73b1df0ceaeeef0f313db557b163bc6b36f32e8a084e54e975a DATABASE_INTEGRITY=ok TWO_CLOUDS=OK LOCAL_STAGING_DELETED=YES
|
|
HEALTH_FILE=backup-audit-v2.json|mtime=2026-09-23 23:47:36.447938462 +0300|sha256=b193b9fc04ed47f247d4c981468387ceff69cf1415e4295ec5b4de9e08d1b6fe
|
|
{"cloud":{"A":{"free":592249131673,"free_fraction":0.538647,"readable":true,"top_dirs":27,"total":1099511627776,"used":507262496103},"B":{"free":592249067814,"free_fraction":0.538647,"readable":true,"top_dirs":27,"total":1099511627776,"used":507262559962}},"cloud_b_restore_drill":true,"dr_proven":true,"generated_at_epoch":1790196452,"issues":[{"code":"RPO_BREACH","logical_id":"vm150","root_cause":"NO_FRESH_V2_OR_SEALED_QUORUM","severity":"HIGH"},{"code":"SOURCE_OFFHOST_MISSING","logical_id":"-","root_cause":"CR_BRANCH_SHA_NOT_EXACT","severity":"MEDIUM"}],"schema":"homelab-backup-audit-v3","source":{"local_head":"6401ea1e99988b9d2550de544c9e45bd7e829bdd","pushurl_disabled":true,"remote_head":null},"status":"RED","targets":[{"decision":"SKIP_FRESH","generation":"app-filebrowser-pve01-20260923T143005Z","kind":"APP","legacy_age_seconds":22640,"logical_id":"app-filebrowser","state":"FRESH"},{"decision":"SKIP_FRESH","generation":"app-hbarhub-pve01-20260923T143158Z","kind":"APP","legacy_age_seconds":22468,"logical_id":"app-hbarhub","state":"FRESH"},{"decision":"SKIP_FRESH","generation":"app-immich-pve01-20260923T143041Z","kind":"APP","legacy_age_seconds":22568,"logical_id":"app-immich","state":"FRESH"},{"decision":"SKIP_FRESH","generation":"app-skladchik-reports-monitor-pve01-20260923T143025Z","kind":"APP","legacy_age_seconds":22618,"logical_id":"app-skladchik-reports-monitor","state":"FRESH"},{"age_seconds":44470,"generation":"control-bootstrap-v2-pve01-20260916T120057Z","kind":"CONTROL","logical_id":"control-bootstrap-v2","state":"FRESH"},{"generation":"v2-ct110-20260923T125546Z-34408ffe4182","kind":"VM","legacy_age_seconds":1696083,"logical_id":"vm110","replica_count":2,"source":"V2","state":"FRESH","state_id":"ct110","v2_age_seconds":28097},{"generation":"v2-ct111-20260923T133222Z-0d3f5fae9c35","kind":"VM","legacy_age_seconds":null,"logical_id":"vm111","replica_count":2,"source":"V2","state":"FRESH","state_id":"ct111","v2_age_seconds":25955},{"generation":"v2-ct112-20260923T200600Z-c1b47cc67dad","kind":"VM","legacy_age_seconds":1734578,"logical_id":"vm112","replica_count":2,"source":"V2","state":"FRESH","state_id":"ct112","v2_age_seconds":2422},{"generation":"v2-ct113-20260923T203800Z-bd913a8e8930","kind":"VM","legacy_age_seconds":null,"logical_id":"vm113","replica_count":2,"source":"V2","state":"FRESH","state_id":"ct113","v2_age_seconds":497},{"generation":"v2-vm130-20260923T011558Z-c9498d37d3d7","kind":"VM","legacy_age_seconds":null,"logical_id":"vm130","replica_count":2,"source":"V2","state":"FRESH","state_id":"vm130","v2_age_seconds":69146},{"generation":"vm150-pve01-20260904T053517Z","kind":"VM","legacy_age_seconds":1696199,"logical_id":"vm150","replica_count":2,"source":"NONE","state":"STALE","state_id":"vm150","v2_age_seconds":74828},{"generation":"v2-vm160-20260923T014608Z-f76d7671af5a","kind":"VM","legacy_age_seconds":null,"logical_id":"vm160","replica_count":2,"source":"V2","state":"FRESH","state_id":"vm160","v2_age_seconds":68046},{"generation":"v2-vm170-20260923T035958Z-923adacbc94b","kind":"VM","legacy_age_seconds":640820,"logical_id":"vm170","replica_count":2,"source":"V2","state":"FRESH","state_id":"vm170","v2_age_seconds":59695},{"generation":"v2-vm171-20260923T021946Z-4d6e4f578075","kind":"VM","legacy_age_seconds":670352,"logical_id":"vm171","replica_count":2,"source":"V2","state":"FRESH","state_id":"vm171","v2_age_seconds":66249},{"generation":"v2-vm190-20260923T032759Z-314d9ba2ec35","kind":"VM","legacy_age_seconds":null,"logical_id":"vm190","replica_count":2,"source":"V2","state":"FRESH","state_id":"vm190","v2_age_seconds":62166},{"generation":"v2-vm9130-20260923T071932Z-c0492db6535b","kind":"VM","legacy_age_seconds":null,"logical_id":"vm9130","replica_count":2,"source":"V2","state":"FRESH","state_id":"vm9130","v2_age_seconds":47261}]}
|
|
HEALTH_FILE=netbird-old-usa-poweroff-observation.txt|mtime=2026-08-22 05:33:01.774623609 +0300|sha256=0307f121dba0190d02875321ed3ca2c433f8a2902678f46814d39cdc476a4a00
|
|
STATUS=OBSERVING
|
|
START_EPOCH=1787365981
|
|
START_UTC=2026-08-22T02:33:01Z
|
|
MIN_SECONDS=86400
|
|
MAX_SECONDS=172800
|
|
NEW_IP=[IP]
|
|
OLD_USA_PROVIDER_DELETION=USER_ATTESTED
|
|
OLD_PUBLIC_IP_REACHABILITY=NOT_USED_AS_HOST_IDENTITY
|
|
NEW_OPERATOR_KEY_ONLY=PROVEN
|
|
NEW_NETBIRD_STACK=3_OF_3
|
|
NEW_PUBLIC_PLANE=GREEN
|
|
FINAL_RETIREMENT=PENDING_OBSERVATION
|
|
HEALTH_FILE=crypto-portfolio-remove-grt-prechange-backup.txt|mtime=2026-07-13 08:10:44.727781612 +0300|sha256=59de8a4e38310ef8236d167be536aa2f0f1d0fa3dd0ad3ac7fd49e67ed6206f3
|
|
STATUS=OK TS=2026-07-13T05:10:13Z TYPE=crypto-portfolio-remove-grt-prechange-backup BAD=0 COMMAND_PHYSICAL_LINES=1 PRODUCTION_CHANGED=ROLLBACK_SNAPSHOT_AND_BACKUP_ONLY PRODUCTION_DATABASE_CHANGED=NO DESIRED_STATE_CHANGED=NO_BY_THIS_COMMAND SERVICE_RESTARTED=NO PRODUCTION_CONTAINER_RECREATED=NO CLOUD_WRITE_RUN=YES_CONTROLLED_PRECHANGE_BACKUP RETENTION_RUN=NO PRODUCTION_APPLY=NO TARGET_HOST=core-apps.gram1.ru DESIRED_STATE_HEAD=80763a6c62b94cde4f3293f3b529980f76919d25 CANDIDATE_ROOT=/var/lib/homelab-change-sets/crypto-portfolio-remove-grt-candidate-v3/20260713T042046Z INPUT_APP_SHA256=684ad884119554503db0ed36252aeb017c0c04b54729f30c51e66a7ab293f8bd INPUT_WORKER_SHA256=40a1c81111d7097120125b140683a8604abcb4051db5073ab8af708a11854038 INPUT_INDEX_SHA256=2b1ab459cabf3050af1db4a7b293fff7c2a9ac18bf4a14faa1efc70a3a3f63e3 EXACT_LOCAL_ROLLBACK=OK ROLLBACK_PATH=/var/lib/homelab-incidents/crypto-remove-grt-prechange-20260713T051013Z ROLLBACK_MANIFEST_SHA256=6cc505f7e13a02b0ddfc9fd3a9700903d7aaf70d8c8ae3f630a4fea69aeb4b1f ROLLBACK_DATABASE_INTEGRITY=ok ROLLBACK_GRT_ROWS=1|1|729 TWO_CLOUD_BACKUP=OK BACKUP_JOB=homelab-pve01-app-crypto-portfolio-20260713T051015Z BACKUP_ARCHIVE_SIZE_BYTES=172776 BACKUP_FILE_COUNT=19 BACKUP_DATABASE_INTEGRITY=ok BACKUP_LOCAL_STAGING_DELETED=YES RESTORE_CHECK=OK_BOTH_CLOUDS PRODUCTION_SOURCE_HASHES_UNCHANGED=YES PRODUCTION_CONTAINER_IDS_AND_RESTARTS_UNCHANGED=YES PRODUCTION_DATABASE_INTEGRITY=ok PRODUCTION_GRT_STATE_PRESERVED=YES PUBLIC_PRODUCTION_STATE_PRESERVED=YES READY_FOR_GRT_PRODUCTION_APPLY=YES PRECHANGE_ROOT=/var/lib/homelab-change-sets/crypto-portfolio-remove-grt-prechange-backup/20260713T051013Z
|
|
HEALTH_FILE=beszel-backup.txt|mtime=2026-07-09 11:38:36.118613080 +0300|sha256=951a25a876714a6b36934a96ae7f85bab06b480633c052b0850ba02d5e0e106c
|
|
STATUS=OK TS=2026-07-09T08:38:10Z TYPE=beszel-backup JOB=homelab-pve01-app-beszel-20260709T083810Z SOURCE=beszel_stack_data SIZE=20030 SHA256=a3d7069dce01b7e7b336268cfe383dc1b0842a974bd5dc9f455442ed86c912e4 LOCAL_BACKUP_DELETED=true NOTE=service-readiness-backup-proof
|
|
HEALTH_FILE=gotify-backup.txt|mtime=2026-07-09 23:39:37.911441599 +0300|sha256=e86e933ad71df820106148796cdccbde701c986034c7e49684f3d9e84bc0d17f
|
|
STATUS=OK TS=2026-07-09T20:39:29Z TYPE=gotify-backup BAD=0 APP=gotify JOB=homelab-pve01-app-gotify-20260709T072202Z SIZE=44431 LOCAL_BACKUP_DELETED=true JSON=/var/lib/homelab-backup/apps/latest-gotify.json
|
|
HEALTH_FILE=minio-console-backup.txt|mtime=2026-07-09 15:13:16.876111706 +0300|sha256=e4965b27706d848070d5ca6c2e974a14c520465f3b1575fda1ab671d6cf0e124
|
|
STATUS=OK TS=2026-07-09T12:13:14Z TYPE=minio-backup JOB=homelab-pve01-app-minio-20260709T051713Z SOURCE=latest-appbackup SIZE=4585 LOCAL_BACKUP_DELETED=true JSON=/var/lib/homelab-backup/apps/latest-minio.json NOTE=service-readiness-backup-proof
|
|
HEALTH_FILE=netbird-vps-restore.txt|mtime=2026-06-17 10:54:28.050678269 +0300|sha256=800a5462a6173c8749059462e39a3263cae2b4db012c1dc3ad336aa3429ab2cb
|
|
STATUS=OK
|
|
SERVICE=netbird-vps-restore-validation
|
|
TIMESTAMP_UTC=20260617T075428Z
|
|
LATEST=20260617T074922Z
|
|
ARCHIVE=/mnt/staging/netbird-vps-backups/snapshots/20260617T074922Z/netbird-vps-backup.tgz
|
|
SIZE=32M
|
|
SHA256=8e0c6538268b4e5dc463e008e59acb9d77873873915bec74ac2a85a9d1523a84
|
|
TOP_ARCHIVE_ENTRIES=9
|
|
ROOT_NETBIRD_ENTRIES=7
|
|
CHECKS=compose,config,dashboard_env
|
|
HEALTH_FILE=bookstack-backup.txt|mtime=2026-07-09 20:58:33.528404671 +0300|sha256=80a1b54e2027c39a6b2c298e558d1352b976e0cd1795cf8f26ae9249b8c2e1dd
|
|
STATUS=OK TS=2026-07-09T17:58:30Z TYPE=bookstack-backup BAD=0 APP=bookstack JOB=homelab-pve01-app-bookstack-20260709T024714Z SIZE=18108 LOCAL_BACKUP_DELETED=true JSON=/var/lib/homelab-backup/apps/latest-bookstack.json
|
|
HEALTH_FILE=backup-last-run.log|mtime=2026-07-24 07:04:41.713444052 +0300|sha256=8618b97d2ffa65b7cc2218710c2add4283285038eaca633a7bffbda898d1e5c7
|
|
===== HOMELAB BACKUP HEALTHCHECK =====
|
|
2026-07-24T07:04:28+03:00
|
|
|
|
LATEST=/mnt/staging/vzdump/auto-full-backup-20260719T004537Z
|
|
AGE_HOURS=123
|
|
MAX_AGE_HOURS=192
|
|
|
|
===== REQUIRED FILES =====
|
|
OK: pve01/vzdump-lxc-110-*.tar.zst
|
|
OK: pve01/vzdump-lxc-112-*.tar.zst
|
|
OK: pve01/vzdump-qemu-150-*.vma.zst
|
|
OK: pve02/vzdump-lxc-111-*.tar.zst
|
|
OK: pve02/vzdump-lxc-113-*.tar.zst
|
|
OK: pve03/vzdump-qemu-130-*.vma.zst
|
|
OK: MASTER_SHA256SUMS.txt
|
|
OK: README_AUTO_FULL_BACKUP.txt
|
|
|
|
===== SIZE CHECK =====
|
|
SIZE_BYTES=19102676192
|
|
SIZE_GB_APPROX=17
|
|
|
|
===== SHA256 CHECK =====
|
|
pve01/SHA256SUMS.txt: OK
|
|
pve01/vzdump-lxc-110-2026_07_19-03_45_38.log: OK
|
|
pve01/vzdump-lxc-110-2026_07_19-03_45_38.tar.zst: OK
|
|
pve01/vzdump-lxc-112-2026_07_19-03_45_47.log: OK
|
|
pve01/vzdump-lxc-112-2026_07_19-03_45_47.tar.zst: OK
|
|
pve01/vzdump-qemu-150-2026_07_19-03_45_58.log: OK
|
|
pve01/vzdump-qemu-150-2026_07_19-03_45_58.vma.zst: OK
|
|
pve02/SHA256SUMS.txt: OK
|
|
pve02/vzdump-lxc-111-2026_07_19-03_47_26.log: OK
|
|
pve02/vzdump-lxc-111-2026_07_19-03_47_26.tar.zst: OK
|
|
pve02/vzdump-lxc-113-2026_07_19-03_47_43.log: OK
|
|
pve02/vzdump-lxc-113-2026_07_19-03_47_43.tar.zst: OK
|
|
pve03/SHA256SUMS.txt: OK
|
|
pve03/vzdump-qemu-130-2026_07_19-03_48_01.log: OK
|
|
pve03/vzdump-qemu-130-2026_07_19-03_48_01.vma.zst: OK
|
|
README_AUTO_FULL_BACKUP.txt: OK
|
|
|
|
===== STORAGE SPACE =====
|
|
Filesystem Type Size Used Avail Use% Mounted on
|
|
/dev/mapper/pve-homelab--staging ext4 295G 148G 132G 53% /mnt/staging
|
|
/dev/mapper/pve-root ext4 98G 41G 53G 44% /
|
|
|
|
STATUS=OK
|
|
REASON=latest backup exists, fresh, complete, and sha256 verified
|
|
HEALTH_FILE=cluster-admin-stage2-preapply-backup.txt|mtime=2026-07-13 04:24:04.995755658 +0300|sha256=82a0a705c46b2cf757dc37ce6a06a981c4cbe3ea3d580459ac7f8c6ae17a153d
|
|
STATUS=OK TS=2026-07-13T01:23:18Z TYPE=cluster-admin-stage2-preapply-backup BAD=0 PREAPPLY_GENERATION=V2_DYNAMIC PRODUCTION_CHANGED=BACKUP_ARTIFACTS_ONLY PRODUCTION_DATABASE_CHANGED=NO PRODUCTION_SCHEMA_CHANGED=NO INCIDENT_ENGINE_RESTARTED=NO COLLECTOR_ONESHOT_RUN=NO CLOUD_WRITE_RUN=YES_CONTROLLED_BACKUP_ONLY ROLLBACK_POINT=VERIFIED DIRECT_DB_SNAPSHOT=VERIFIED DIRECT_DB_RESTORE_TEST=OK DIRECT_DB_DUMP_SIZE_BYTES=1818083 DIRECT_DB_DUMP_SHA256=c0fe2161bb6e61b423bc72d809bdd579ac29dc4f7bac79ba4c493bfa9f8b1d5e PRECHANGE_COLLECTOR_SHA256=a83abde2290d8392b1c631122671bcb680f9271d0b576fafacb8876d3c8034f1 CANDIDATE_COLLECTOR_SHA256=5dda66a1878046223af9c28a393129b44991a8083b430a76302767b460e7cbbe FOUNDATION_COUNTS=84|21|1|227|26|0 DYNAMIC_INVARIANTS=0|0|21|21|0|0|0|0|0|0 RETIRED_PLACEHOLDERS_ALLOWED=YES FIXED_TOTAL_ENTITIES_REQUIRED=NO FIXED_TOTAL_ALIASES_REQUIRED=NO APPBACKUP_JOB=homelab-pve01-app-cluster-admin-incident-engine-20260713T012320Z APPBACKUP_STATUS=success APPBACKUP_RESTORE_CHECK=OK APPBACKUP_ROUTING_DOCKGE_UNCHANGED=YES MIGRATION_PATCH_REQUIRED=NO COLLECTOR_PATCH_REQUIRED=NO APPLY_VERIFIER_PATCH_REQUIRED=YES PREFLIGHT_TRANSACTION_ROLLBACK_REQUIRED=YES ROLLBACK_AFTER_COMMITTED_SYNC=PRESERVE_APPLIED_STATE_AND_DIAGNOSE PRODUCTION_UNCHANGED=YES OVERALL=OK DUTY_ADMIN=OK READY_FOR_DYNAMIC_PRODUCTION_APPLY=YES DESIRED_STATE_HEAD=4d0d03395d2baf4024ad7c9d5870f21bdaee9f99 PREAPPLY_ROOT=/var/lib/homelab-stage2-preapply-v2/20260713T012318Z REMOTE_SNAPSHOT_DIR=/var/lib/cluster-admin-stage2-preapply-v2/20260713T012318Z CONTRACT=/var/lib/homelab-change-sets/stage2-dynamic-redryrun/20260713T012006Z/dynamic-acceptance-contract.json EVIDENCE_MANIFEST=/var/lib/homelab-stage2-preapply-v2/20260713T012318Z/SHA256SUMS
|
|
HEALTH_FILE=blackbox-exporter-two-cloud-backup.txt|mtime=2026-07-10 22:20:26.139732318 +0300|sha256=ab822f689cc4b7c38a0eed3c6a3164993ceb54fa05033250c68d934d3df0f259
|
|
STATUS=OK TS=2026-07-10T19:20:26Z TYPE=blackbox-exporter-two-cloud-backup BAD=0 JOB=homelab-pve01-app-blackbox-exporter-20260710T191958Z ARCHIVE_SIZE_BYTES=592 LOCAL_STAGING_DELETED=true OFFHOST=OK RESTORE=OK
|
|
HEALTH_FILE=backup.json|mtime=2026-07-24 07:04:41.723802122 +0300|sha256=550e7a326c46e3e765a6db01da2ddcaddd83808adfa638d980685286bebb45cb
|
|
{
|
|
"status": "OK",
|
|
"reason": "latest backup exists, fresh, complete, and sha256 verified",
|
|
"latest": "/mnt/staging/vzdump/auto-full-backup-20260719T004537Z",
|
|
"age_hours": "123",
|
|
"start": "2026-07-24T07:04:28+03:00",
|
|
"end": "2026-07-24T07:04:41+03:00",
|
|
"rc": 0
|
|
}
|
|
HEALTH_FILE=cluster-admin-backup-webpanel-final-seal.txt|mtime=2026-07-09 08:24:54.072015305 +0300|sha256=9739e2a0d2bbb859b25255fd7acae984ee05a16fb79cd9e59ff81529c4acf3fd
|
|
STATUS=OK TS=2026-07-09T05:24:54Z TYPE=cluster-admin-backup-webpanel-final-seal BAD=0 VMID=180 URL=http://[IP]:8080/ BACKUP=MAIL_CLOUD_2COPY_VERIFY_OK WEBPANEL=OK WEBPANEL_SYNC=OK OVERALL_HARDGATE=YES FINAL_READINESS_HARDGATE=YES FAILED_UNITS=pve01:0,pve02:0,pve03:0,edge:0 DIRTY_COUNT=0 REPORT=/var/lib/homelab-ideal-cluster/cluster-admin-backup-webpanel-final-seal-latest.md
|
|
HEALTH_FILE=immich-backup.txt|mtime=2026-07-09 23:09:25.074550210 +0300|sha256=8ef28f515f12888a7b1ff5b949d09b0eef90c1fc11da523b388ddd905167b52d
|
|
STATUS=OK TS=2026-07-09T20:09:12Z TYPE=immich-backup BAD=0 APP=immich JOB=homelab-pve01-app-immich-20260709T072002Z SIZE=328392286 LOCAL_BACKUP_DELETED=true JSON=/var/lib/homelab-backup/apps/latest-immich.json MEDIA_BACKUP_HEALTH=/var/lib/homelab-health/immich-media-backup.txt
|
|
HEALTH_FILE=grafana-backup.txt|mtime=2026-07-09 10:46:00.905608392 +0300|sha256=1347e0bba67dd249caa8d4e46faf4d6bb2fd573255c1f78449995f97eea30014
|
|
STATUS=OK TS=2026-07-09T07:46:00Z TYPE=grafana-backup JOB=homelab-pve01-app-grafana-20260709T074600Z SOURCE=grafana_sqlite_backup_plus_provisioning SIZE=129929 SHA256=582537912f306666f04d9e855491f7275bdb31b39b4c9b0156444a297f8c1641 LOCAL_BACKUP_DELETED=true NOTE=service-readiness-backup-proof
|
|
HEALTH_FILE=appbackupctl-routing.txt|mtime=2026-07-13 03:29:40.002577636 +0300|sha256=d8ddbdb46b42bcefb4f03b50ef4b88bdc5e3cd1cea087dd035fd90fd78e1a252
|
|
STATUS=OK TS=2026-07-13T00:29:36Z TYPE=appbackupctl-routing BAD=0 RESULT=APPLIED PRODUCTION_CHANGED=BACKUP_CLI_ROUTING_AND_DESIRED_STATE_ONLY PRODUCTION_DATABASE_CHANGED=NO PRODUCTION_SCHEMA_CHANGED=NO SERVICE_RESTARTED=NO CLOUD_WRITE_RUN=NO DEFAULT_ROUTE=dockge-stacks CLUSTER_ADMIN_ROUTE=homelab-cluster-admin-incident-engine-backup.service UNSUPPORTED_TARGET_POLICY=REFUSE_RC_2 ROUTE_TEST_DOCKGE=OK ROUTE_TEST_CLUSTER_ADMIN=OK ROUTE_TEST_DEFAULT=OK ROUTE_TEST_UNSUPPORTED=OK FRAMEWORK_CHECK=OK UNSUPPORTED_ROUTE_BACKUP_CREATED=NO LIVE_MATCH_DESIRED_STATE=YES PVE01_FAILED_UNITS=0 OVERALL=OK DUTY_ADMIN=OK STAGE1_CLOSED=YES STAGE2_READY=YES OLD_HEAD=bd5eb856a8d61a5fb83c2bcaf74cb8ac98f075c5 NEW_HEAD=4d0d03395d2baf4024ad7c9d5870f21bdaee9f99 CHANGE_SET=/var/lib/homelab-change-sets/appbackupctl-routing/20260713T002936Z
|
|
HEALTH_FILE=vaultwarden-backup.txt|mtime=2026-08-30 21:08:08.480614603 +0300|sha256=ba016970c3d244955b82977178290fea97126dbcc2a44e4758ecd28ad720a499
|
|
STATUS=FAIL
|
|
TS=2026-08-30T18:08:08Z
|
|
TYPE=backup-matrix-generated
|
|
JOB_ID=homelab_appbackup_sqlite_batch
|
|
JOB_NAME=homelab-appbackup-sqlite-batch
|
|
MATRIX_MODE=authoritative
|
|
TIMER=homelab-appbackup-sqlite-batch.timer
|
|
SERVICE=homelab-appbackup-sqlite-batch.service
|
|
TIMER_ENABLED=not-found
|
|
TIMER_ACTIVE=inactive
|
|
SOURCE_KIND=systemd
|
|
EVIDENCE_LEVEL=execution_only
|
|
SERVICE_RESULT=success
|
|
SERVICE_EXEC_MAIN_STATUS=0
|
|
SOURCE_HEALTH=systemd:homelab-appbackup-sqlite-batch.service
|
|
SOURCE_STATUS=OK
|
|
SOURCE_MTIME_UTC=2026-07-24T01:46:28Z
|
|
AGE_SECONDS=3255700
|
|
MAX_AGE_SECONDS=172800
|
|
FRESHNESS=STALE
|
|
LEGACY_SOURCE_PATH=/var/lib/homelab-health/vaultwarden-backup.txt
|
|
CANONICAL_JOB_ID=homelab_appbackup_sqlite_batch
|
|
CANONICAL_STATE=/var/lib/homelab-backup-matrix/state.json
|
|
HEALTH_FILE=hapusya-site-preapply-backup.txt|mtime=2026-07-13 14:03:06.015223430 +0300|sha256=00e05707281a8d6e841ec0f6d2b8c95ee0f9fbc090ab90b718e85530d41854f8
|
|
STATUS=OK TS=2026-07-13T11:03:06Z TYPE=hapusya-site-preapply-backup BAD=0 NPMPLUS_JOB=homelab-pve01-app-npmplus-edge-20260713T110238Z TWO_CLOUDS=OK RESTORE_CHECK=OK ROLLBACK=/var/lib/homelab-incidents/hapusya-site-preapply-20260713T110305Z ROLLBACK_MANIFEST_SHA256=6b4020425c5d6c581c48a5dc95b20d54fd7906468e4ba8d27c0f0325e2df81a3 CANDIDATE_BROWSER=OK BROWSER_RUNS=4 PRODUCTION_CHANGED=NO
|
|
HEALTH_FILE=pve01-backup.txt|mtime=2026-07-09 15:29:41.985138298 +0300|sha256=2b8ca75eeec121d7e2b0c6a309d43798da5f864b57fef327f37a56b9ced3b5a8
|
|
STATUS=OK TS=2026-07-09T12:29:38Z TYPE=proxmox-pve01-backup JOB=homelab-pve01-app-proxmox-pve01-20260709T122938Z SOURCE=safe_pve01_config_facts SIZE=4314 LOCAL_BACKUP_DELETED=true JSON=/var/lib/homelab-backup/apps/latest-proxmox-pve01.json NOTE=service-readiness-backup-proof
|
|
HEALTH_FILE=backup-observability-history.jsonl|mtime=2026-09-21 07:07:33.804291373 +0300|sha256=24fa8197552d64778c83071fb94a64accd398075ce5d1e675dcdf6c97a84c36b
|
|
{"clouds":[{"free_bytes":792267937635,"remote":"pve01-mail-01-crypt","total_bytes":1099511627776,"used_bytes":307243690141},{"free_bytes":792267873776,"remote":"pve01-mail-02-crypt","total_bytes":1099511627776,"used_bytes":307243754000}],"observed_at_epoch":1788438952,"schema":"homelab-backup-observability-snapshot-v1","targets":[{"archive_size_bytes":474143241,"id":"110","kind":"VM","logical_id":"vm110"},{"archive_size_bytes":369191552,"id":"111","kind":"VM","logical_id":"vm111"},{"archive_size_bytes":223627414,"id":"112","kind":"VM","logical_id":"vm112"},{"archive_size_bytes":221521137,"id":"113","kind":"VM","logical_id":"vm113"},{"archive_size_bytes":10622796310,"id":"130","kind":"VM","logical_id":"vm130"},{"archive_size_bytes":890940072,"id":"150","kind":"VM","logical_id":"vm150"},{"archive_size_bytes":3469023634,"id":"160","kind":"VM","logical_id":"vm160"},{"archive_size_bytes":7824247513,"id":"170","kind":"VM","logical_id":"vm170"},{"archive_size_bytes":2026787893,"id":"171","kind":"VM","logical_id":"vm171"},{"archive_size_bytes":1176306536,"id":"180","kind":"VM","logical_id":"vm180"},{"archive_size_bytes":1551688736,"id":"190","kind":"VM","logical_id":"vm190"},{"archive_size_bytes":11005893463,"id":"9130","kind":"VM","logical_id":"vm9130"},{"archive_size_bytes":383573170,"id":"200","kind":"VM","logical_id":"vm200"},{"archive_size_bytes":3193,"id":"filebrowser","kind":"APP","logical_id":"app-filebrowser"},{"archive_size_bytes":152745,"id":"skladchik-reports-monitor","kind":"APP","logical_id":"app-skladchik-reports-monitor"},{"archive_size_bytes":270193536,"id":"immich","kind":"APP","logical_id":"app-immich"},{"archive_size_bytes":75298798,"id":"hbarhub","kind":"APP","logical_id":"app-hbarhub"},{"archive_size_bytes":8959200,"id":"control-bootstrap-v2","kind":"CONTROL","logical_id":"control-bootstrap-v2"}]}
|
|
{"clouds":[{"free_bytes":792267937635,"remote":"pve01-mail-01-crypt","total_bytes":1099511627776,"used_bytes":307243690141},{"free_bytes":792267873776,"remote":"pve01-mail-02-crypt","total_bytes":1099511627776,"used_bytes":307243754000}],"observed_at_epoch":1788442632,"schema":"homelab-backup-observability-snapshot-v1","targets":[{"archive_size_bytes":474143241,"id":"110","kind":"VM","logical_id":"vm110"},{"archive_size_bytes":369191552,"id":"111","kind":"VM","logical_id":"vm111"},{"archive_size_bytes":223627414,"id":"112","kind":"VM","logical_id":"vm112"},{"archive_size_bytes":221521137,"id":"113","kind":"VM","logical_id":"vm113"},{"archive_size_bytes":10622796310,"id":"130","kind":"VM","logical_id":"vm130"},{"archive_size_bytes":890940072,"id":"150","kind":"VM","logical_id":"vm150"},{"archive_size_bytes":3469023634,"id":"160","kind":"VM","logical_id":"vm160"},{"archive_size_bytes":7824247513,"id":"170","kind":"VM","logical_id":"vm170"},{"archive_size_bytes":2026787893,"id":"171","kind":"VM","logical_id":"vm171"},{"archive_size_bytes":1176306536,"id":"180","kind":"VM","logical_id":"vm180"},{"archive_size_bytes":1551688736,"id":"190","kind":"VM","logical_id":"vm190"},{"archive_size_bytes":11005893463,"id":"9130","kind":"VM","logical_id":"vm9130"},{"archive_size_bytes":383573170,"id":"200","kind":"VM","logical_id":"vm200"},{"archive_size_bytes":3193,"id":"filebrowser","kind":"APP","logical_id":"app-filebrowser"},{"archive_size_bytes":152745,"id":"skladchik-reports-monitor","kind":"APP","logical_id":"app-skladchik-reports-monitor"},{"archive_size_bytes":270193536,"id":"immich","kind":"APP","logical_id":"app-immich"},{"archive_size_bytes":75298798,"id":"hbarhub","kind":"APP","logical_id":"app-hbarhub"},{"archive_size_bytes":8959200,"id":"control-bootstrap-v2","kind":"CONTROL","logical_id":"control-bootstrap-v2"}]}
|
|
{"clouds":[{"free_bytes":792267937635,"remote":"pve01-mail-01-crypt","total_bytes":1099511627776,"used_bytes":307243690141},{"free_bytes":792267873776,"remote":"pve01-mail-02-crypt","total_bytes":1099511627776,"used_bytes":307243754000}],"observed_at_epoch":1788447976,"schema":"homelab-backup-observability-snapshot-v1","targets":[{"archive_size_bytes":474143241,"id":"110","kind":"VM","logical_id":"vm110"},{"archive_size_bytes":369191552,"id":"111","kind":"VM","logical_id":"vm111"},{"archive_size_bytes":223627414,"id":"112","kind":"VM","logical_id":"vm112"},{"archive_size_bytes":221521137,"id":"113","kind":"VM","logical_id":"vm113"},{"archive_size_bytes":10622796310,"id":"130","kind":"VM","logical_id":"vm130"},{"archive_size_bytes":890940072,"id":"150","kind":"VM","logical_id":"vm150"},{"archive_size_bytes":3469023634,"id":"160","kind":"VM","logical_id":"vm160"},{"archive_size_bytes":7824247513,"id":"170","kind":"VM","logical_id":"vm170"},{"archive_size_bytes":2026787893,"id":"171","kind":"VM","logical_id":"vm171"},{"archive_size_bytes":1176306536,"id":"180","kind":"VM","logical_id":"vm180"},{"archive_size_bytes":1551688736,"id":"190","kind":"VM","logical_id":"vm190"},{"archive_size_bytes":11005893463,"id":"9130","kind":"VM","logical_id":"vm9130"},{"archive_size_bytes":383573170,"id":"200","kind":"VM","logical_id":"vm200"},{"archive_size_bytes":3193,"id":"filebrowser","kind":"APP","logical_id":"app-filebrowser"},{"archive_size_bytes":152745,"id":"skladchik-reports-monitor","kind":"APP","logical_id":"app-skladchik-reports-monitor"},{"archive_size_bytes":270193536,"id":"immich","kind":"APP","logical_id":"app-immich"},{"archive_size_bytes":75298798,"id":"hbarhub","kind":"APP","logical_id":"app-hbarhub"},{"archive_size_bytes":8959200,"id":"control-bootstrap-v2","kind":"CONTROL","logical_id":"control-bootstrap-v2"}]}
|
|
{"clouds":[{"free_bytes":792267937635,"remote":"pve01-mail-01-crypt","total_bytes":1099511627776,"used_bytes":307243690141},{"free_bytes":792267873776,"remote":"pve01-mail-02-crypt","total_bytes":1099511627776,"used_bytes":307243754000}],"observed_at_epoch":1788451586,"schema":"homelab-backup-observability-snapshot-v1","targets":[{"archive_size_bytes":474143241,"id":"110","kind":"VM","logical_id":"vm110"},{"archive_size_bytes":369191552,"id":"111","kind":"VM","logical_id":"vm111"},{"archive_size_bytes":223627414,"id":"112","kind":"VM","logical_id":"vm112"},{"archive_size_bytes":221521137,"id":"113","kind":"VM","logical_id":"vm113"},{"archive_size_bytes":10622796310,"id":"130","kind":"VM","logical_id":"vm130"},{"archive_size_bytes":890940072,"id":"150","kind":"VM","logical_id":"vm150"},{"archive_size_bytes":3469023634,"id":"160","kind":"VM","logical_id":"vm160"},{"archive_size_bytes":7824247513,"id":"170","kind":"VM","logical_id":"vm170"},{"archive_size_bytes":2026787893,"id":"171","kind":"VM","logical_id":"vm171"},{"archive_size_bytes":1176306536,"id":"180","kind":"VM","logical_id":"vm180"},{"archive_size_bytes":1551688736,"id":"190","kind":"VM","logical_id":"vm190"},{"archive_size_bytes":11005893463,"id":"9130","kind":"VM","logical_id":"vm9130"},{"archive_size_bytes":383573170,"id":"200","kind":"VM","logical_id":"vm200"},{"archive_size_bytes":3193,"id":"filebrowser","kind":"APP","logical_id":"app-filebrowser"},{"archive_size_bytes":152745,"id":"skladchik-reports-monitor","kind":"APP","logical_id":"app-skladchik-reports-monitor"},{"archive_size_bytes":270193536,"id":"immich","kind":"APP","logical_id":"app-immich"},{"archive_size_bytes":75298798,"id":"hbarhub","kind":"APP","logical_id":"app-hbarhub"},{"archive_size_bytes":8959200,"id":"control-bootstrap-v2","kind":"CONTROL","logical_id":"control-bootstrap-v2"}]}
|
|
{"clouds":[{"free_bytes":771612988784,"remote":"pve01-mail-01-crypt","total_bytes":1099511627776,"used_bytes":327898638992},{"free_bytes":771612924925,"remote":"pve01-mail-02-crypt","total_bytes":1099511627776,"used_bytes":327898702851}],"observed_at_epoch":1788456981,"schema":"homelab-backup-observability-snapshot-v1","targets":[{"archive_size_bytes":476345840,"id":"110","kind":"VM","logical_id":"vm110"},{"archive_size_bytes":369191552,"id":"111","kind":"VM","logical_id":"vm111"},{"archive_size_bytes":223627414,"id":"112","kind":"VM","logical_id":"vm112"},{"archive_size_bytes":221521137,"id":"113","kind":"VM","logical_id":"vm113"},{"archive_size_bytes":892033158,"id":"150","kind":"VM","logical_id":"vm150"},{"archive_size_bytes":3469023634,"id":"160","kind":"VM","logical_id":"vm160"},{"archive_size_bytes":7937058725,"id":"170","kind":"VM","logical_id":"vm170"},{"archive_size_bytes":2026787893,"id":"171","kind":"VM","logical_id":"vm171"},{"archive_size_bytes":1176306536,"id":"180","kind":"VM","logical_id":"vm180"},{"archive_size_bytes":1551688736,"id":"190","kind":"VM","logical_id":"vm190"},{"archive_size_bytes":11005893463,"id":"9130","kind":"VM","logical_id":"vm9130"},{"archive_size_bytes":386825665,"id":"200","kind":"VM","logical_id":"vm200"},{"archive_size_bytes":3192,"id":"filebrowser","kind":"APP","logical_id":"app-filebrowser"},{"archive_size_bytes":153389,"id":"skladchik-reports-monitor","kind":"APP","logical_id":"app-skladchik-reports-monitor"},{"archive_size_bytes":270193550,"id":"immich","kind":"APP","logical_id":"app-immich"},{"archive_size_bytes":75379068,"id":"hbarhub","kind":"APP","logical_id":"app-hbarhub"},{"archive_size_bytes":8959200,"id":"control-bootstrap-v2","kind":"CONTROL","logical_id":"control-bootstrap-v2"}]}
|
|
{"clouds":[{"free_bytes":755855450788,"remote":"pve01-mail-01-crypt","total_bytes":1099511627776,"used_bytes":343656176988},{"free_bytes":755855386929,"remote":"pve01-mail-02-crypt","total_bytes":1099511627776,"used_bytes":343656240847}],"observed_at_epoch":1788460613,"schema":"homelab-backup-observability-snapshot-v1","targets":[{"archive_size_bytes":476345840,"id":"110","kind":"VM","logical_id":"vm110"},{"archive_size_bytes":369191552,"id":"111","kind":"VM","logical_id":"vm111"},{"archive_size_bytes":223627414,"id":"112","kind":"VM","logical_id":"vm112"},{"archive_size_bytes":221521137,"id":"113","kind":"VM","logical_id":"vm113"},{"archive_size_bytes":10611899437,"id":"130","kind":"VM","logical_id":"vm130"},{"archive_size_bytes":892033158,"id":"150","kind":"VM","logical_id":"vm150"},{"archive_size_bytes":3469023634,"id":"160","kind":"VM","logical_id":"vm160"},{"archive_size_bytes":7937058725,"id":"170","kind":"VM","logical_id":"vm170"},{"archive_size_bytes":2026047659,"id":"171","kind":"VM","logical_id":"vm171"},{"archive_size_bytes":1171789442,"id":"180","kind":"VM","logical_id":"vm180"},{"archive_size_bytes":1557482946,"id":"190","kind":"VM","logical_id":"vm190"},{"archive_size_bytes":386825665,"id":"200","kind":"VM","logical_id":"vm200"},{"archive_size_bytes":3192,"id":"filebrowser","kind":"APP","logical_id":"app-filebrowser"},{"archive_size_bytes":153389,"id":"skladchik-reports-monitor","kind":"APP","logical_id":"app-skladchik-reports-monitor"},{"archive_size_bytes":270193550,"id":"immich","kind":"APP","logical_id":"app-immich"},{"archive_size_bytes":75379068,"id":"hbarhub","kind":"APP","logical_id":"app-hbarhub"},{"archive_size_bytes":8959200,"id":"control-bootstrap-v2","kind":"CONTROL","logical_id":"control-bootstrap-v2"}]}
|
|
{"clouds":[{"free_bytes":751486275845,"remote":"pve01-mail-01-crypt","total_bytes":1099511627776,"used_bytes":348025351931},{"free_bytes":751486211986,"remote":"pve01-mail-02-crypt","total_bytes":1099511627776,"used_bytes":348025415790}],"observed_at_epoch":1788465932,"schema":"homelab-backup-observability-snapshot-v1","targets":[{"archive_size_bytes":476345840,"id":"110","kind":"VM","logical_id":"vm110"},{"archive_size_bytes":369892675,"id":"111","kind":"VM","logical_id":"vm111"},{"archive_size_bytes":223672749,"id":"112","kind":"VM","logical_id":"vm112"},{"archive_size_bytes":221529138,"id":"113","kind":"VM","logical_id":"vm113"},{"archive_size_bytes":10611899437,"id":"130","kind":"VM","logical_id":"vm130"},{"archive_size_bytes":892033158,"id":"150","kind":"VM","logical_id":"vm150"},{"archive_size_bytes":3553006761,"id":"160","kind":"VM","logical_id":"vm160"},{"archive_size_bytes":7937058725,"id":"170","kind":"VM","logical_id":"vm170"},{"archive_size_bytes":2026047659,"id":"171","kind":"VM","logical_id":"vm171"},{"archive_size_bytes":1171789442,"id":"180","kind":"VM","logical_id":"vm180"},{"archive_size_bytes":1557482946,"id":"190","kind":"VM","logical_id":"vm190"},{"archive_size_bytes":10998362942,"id":"9130","kind":"VM","logical_id":"vm9130"},{"archive_size_bytes":386825665,"id":"200","kind":"VM","logical_id":"vm200"},{"archive_size_bytes":3192,"id":"filebrowser","kind":"APP","logical_id":"app-filebrowser"},{"archive_size_bytes":153389,"id":"skladchik-reports-monitor","kind":"APP","logical_id":"app-skladchik-reports-monitor"},{"archive_size_bytes":270193550,"id":"immich","kind":"APP","logical_id":"app-immich"},{"archive_size_bytes":75379068,"id":"hbarhub","kind":"APP","logical_id":"app-hbarhub"},{"archive_size_bytes":8959200,"id":"control-bootstrap-v2","kind":"CONTROL","logical_id":"control-bootstrap-v2"}]}
|
|
{"clouds":[{"free_bytes":751477313023,"remote":"pve01-mail-01-crypt","total_bytes":1099511627776,"used_bytes":348034314753},{"free_bytes":751477249164,"remote":"pve01-mail-02-crypt","total_bytes":1099511627776,"used_bytes":348034378612}],"observed_at_epoch":1788469583,"schema":"homelab-backup-observability-snapshot-v1","targets":[{"archive_size_bytes":476345840,"id":"110","kind":"VM","logical_id":"vm110"},{"archive_size_bytes":369892675,"id":"111","kind":"VM","logical_id":"vm111"},{"archive_size_bytes":223672749,"id":"112","kind":"VM","logical_id":"vm112"},{"archive_size_bytes":221529138,"id":"113","kind":"VM","logical_id":"vm113"},{"archive_size_bytes":10611899437,"id":"130","kind":"VM","logical_id":"vm130"},{"archive_size_bytes":892033158,"id":"150","kind":"VM","logical_id":"vm150"},{"archive_size_bytes":3553006761,"id":"160","kind":"VM","logical_id":"vm160"},{"archive_size_bytes":7937058725,"id":"170","kind":"VM","logical_id":"vm170"},{"archive_size_bytes":2026047659,"id":"171","kind":"VM","logical_id":"vm171"},{"archive_size_bytes":1171789442,"id":"180","kind":"VM","logical_id":"vm180"},{"archive_size_bytes":1557482946,"id":"190","kind":"VM","logical_id":"vm190"},{"archive_size_bytes":10998362942,"id":"9130","kind":"VM","logical_id":"vm9130"},{"archive_size_bytes":386825665,"id":"200","kind":"VM","logical_id":"vm200"},{"archive_size_bytes":3192,"id":"filebrowser","kind":"APP","logical_id":"app-filebrowser"},{"archive_size_bytes":153389,"id":"skladchik-reports-monitor","kind":"APP","logical_id":"app-skladchik-reports-monitor"},{"archive_size_bytes":270193550,"id":"immich","kind":"APP","logical_id":"app-immich"},{"archive_size_bytes":75379068,"id":"hbarhub","kind":"APP","logical_id":"app-hbarhub"},{"archive_size_bytes":8959291,"id":"control-bootstrap-v2","kind":"CONTROL","logical_id":"control-bootstrap-v2"}]}
|
|
{"clouds":[{"free_bytes":751477313023,"remote":"pve01-mail-01-crypt","total_bytes":1099511627776,"used_bytes":348034314753},{"free_bytes":751477249164,"remote":"pve01-mail-02-crypt","total_bytes":1099511627776,"used_bytes":348034378612}],"observed_at_epoch":1788474989,"schema":"homelab-backup-observability-snapshot-v1","targets":[{"archive_size_bytes":476345840,"id":"110","kind":"VM","logical_id":"vm110"},{"archive_size_bytes":369892675,"id":"111","kind":"VM","logical_id":"vm111"},{"archive_size_bytes":223672749,"id":"112","kind":"VM","logical_id":"vm112"},{"archive_size_bytes":221529138,"id":"113","kind":"VM","logical_id":"vm113"},{"archive_size_bytes":10611899437,"id":"130","kind":"VM","logical_id":"vm130"},{"archive_size_bytes":892033158,"id":"150","kind":"VM","logical_id":"vm150"},{"archive_size_bytes":3553006761,"id":"160","kind":"VM","logical_id":"vm160"},{"archive_size_bytes":7937058725,"id":"170","kind":"VM","logical_id":"vm170"},{"archive_size_bytes":2026047659,"id":"171","kind":"VM","logical_id":"vm171"},{"archive_size_bytes":1171789442,"id":"180","kind":"VM","logical_id":"vm180"},{"archive_size_bytes":1557482946,"id":"190","kind":"VM","logical_id":"vm190"},{"archive_size_bytes":10998362942,"id":"9130","kind":"VM","logical_id":"vm9130"},{"archive_size_bytes":386825665,"id":"200","kind":"VM","logical_id":"vm200"},{"archive_size_bytes":3192,"id":"filebrowser","kind":"APP","logical_id":"app-filebrowser"},{"archive_size_bytes":153389,"id":"skladchik-reports-monitor","kind":"APP","logical_id":"app-skladchik-reports-monitor"},{"archive_size_bytes":270193550,"id":"immich","kind":"APP","logical_id":"app-immich"},{"archive_size_bytes":75379068,"id":"hbarhub","kind":"APP","logical_id":"app-hbarhub"},{"archive_size_bytes":8959291,"id":"control-bootstrap-v2","kind":"CONTROL","logical_id":"control-bootstrap-v2"}]}
|
|
{"clouds":[{"free_bytes":751477313023,"remote":"pve01-mail-01-crypt","total_bytes":1099511627776,"used_bytes":348034314753},{"free_bytes":751477249164,"remote":"pve01-mail-02-crypt","total_bytes":1099511627776,"used_bytes":348034378612}],"observed_at_epoch":1788478603,"schema":"homelab-backup-observability-snapshot-v1","targets":[{"archive_size_bytes":476345840,"id":"110","kind":"VM","logical_id":"vm110"},{"archive_size_bytes":369892675,"id":"111","kind":"VM","logical_id":"vm111"},{"archive_size_bytes":223672749,"id":"112","kind":"VM","logical_id":"vm112"},{"archive_size_bytes":221529138,"id":"113","kind":"VM","logical_id":"vm113"},{"archive_size_bytes":10611899437,"id":"130","kind":"VM","logical_id":"vm130"},{"archive_size_bytes":892033158,"id":"150","kind":"VM","logical_id":"vm150"},{"archive_size_bytes":3553006761,"id":"160","kind":"VM","logical_id":"vm160"},{"archive_size_bytes":7937058725,"id":"170","kind":"VM","logical_id":"vm170"},{"archive_size_bytes":2026047659,"id":"171","kind":"VM","logical_id":"vm171"},{"archive_size_bytes":1171789442,"id":"180","kind":"VM","logical_id":"vm180"},{"archive_size_bytes":1557482946,"id":"190","kind":"VM","logical_id":"vm190"},{"archive_size_bytes":10998362942,"id":"9130","kind":"VM","logical_id":"vm9130"},{"archive_size_bytes":386825665,"id":"200","kind":"VM","logical_id":"vm200"},{"archive_size_bytes":3192,"id":"filebrowser","kind":"APP","logical_id":"app-filebrowser"},{"archive_size_bytes":153389,"id":"skladchik-reports-monitor","kind":"APP","logical_id":"app-skladchik-reports-monitor"},{"archive_size_bytes":270193550,"id":"immich","kind":"APP","logical_id":"app-immich"},{"archive_size_bytes":75379068,"id":"hbarhub","kind":"APP","logical_id":"app-hbarhub"},{"archive_size_bytes":8959291,"id":"control-bootstrap-v2","kind":"CONTROL","logical_id":"control-bootstrap-v2"}]}
|
|
{"clouds":[{"free_bytes":751477313023,"remote":"pve01-mail-01-crypt","total_bytes":1099511627776,"used_bytes":348034314753},{"free_bytes":751477249164,"remote":"pve01-mail-02-crypt","total_bytes":1099511627776,"used_bytes":348034378612}],"observed_at_epoch":1788482209,"schema":"homelab-backup-observability-snapshot-v1","targets":[{"archive_size_bytes":476345840,"id":"110","kind":"VM","logical_id":"vm110"},{"archive_size_bytes":369892675,"id":"111","kind":"VM","logical_id":"vm111"},{"archive_size_bytes":223672749,"id":"112","kind":"VM","logical_id":"vm112"},{"archive_size_bytes":221529138,"id":"113","kind":"VM","logical_id":"vm113"},{"archive_size_bytes":10611899437,"id":"130","kind":"VM","logical_id":"vm130"},{"archive_size_bytes":892033158,"id":"150","kind":"VM","logical_id":"vm150"},{"archive_size_bytes":3553006761,"id":"160","kind":"VM","logical_id":"vm160"},{"archive_size_bytes":7937058725,"id":"170","kind":"VM","logical_id":"vm170"},{"archive_size_bytes":2026047659,"id":"171","kind":"VM","logical_id":"vm171"},{"archive_size_bytes":1171789442,"id":"180","kind":"VM","logical_id":"vm180"},{"archive_size_bytes":1557482946,"id":"190","kind":"VM","logical_id":"vm190"},{"archive_size_bytes":10998362942,"id":"9130","kind":"VM","logical_id":"vm9130"},{"archive_size_bytes":386825665,"id":"200","kind":"VM","logical_id":"vm200"},{"archive_size_bytes":3192,"id":"filebrowser","kind":"APP","logical_id":"app-filebrowser"},{"archive_size_bytes":153389,"id":"skladchik-reports-monitor","kind":"APP","logical_id":"app-skladchik-reports-monitor"},{"archive_size_bytes":270193550,"id":"immich","kind":"APP","logical_id":"app-immich"},{"archive_size_bytes":75379068,"id":"hbarhub","kind":"APP","logical_id":"app-hbarhub"},{"archive_size_bytes":8959291,"id":"control-bootstrap-v2","kind":"CONTROL","logical_id":"control-bootstrap-v2"}]}
|
|
{"clouds":[{"free_bytes":751477313023,"remote":"pve01-mail-01-crypt","total_bytes":1099511627776,"used_bytes":348034314753},{"free_bytes":751477249164,"remote":"pve01-mail-02-crypt","total_bytes":1099511627776,"used_bytes":348034378612}],"observed_at_epoch":1788487523,"schema":"homelab-backup-observability-snapshot-v1","targets":[{"archive_size_bytes":476345840,"id":"110","kind":"VM","logical_id":"vm110"},{"archive_size_bytes":369892675,"id":"111","kind":"VM","logical_id":"vm111"},{"archive_size_bytes":223672749,"id":"112","kind":"VM","logical_id":"vm112"},{"archive_size_bytes":221529138,"id":"113","kind":"VM","logical_id":"vm113"},{"archive_size_bytes":10611899437,"id":"130","kind":"VM","logical_id":"vm130"},{"archive_size_bytes":892033158,"id":"150","kind":"VM","logical_id":"vm150"},{"archive_size_bytes":3553006761,"id":"160","kind":"VM","logical_id":"vm160"},{"archive_size_bytes":7937058725,"id":"170","kind":"VM","logical_id":"vm170"},{"archive_size_bytes":2026047659,"id":"171","kind":"VM","logical_id":"vm171"},{"archive_size_bytes":1171789442,"id":"180","kind":"VM","logical_id":"vm180"},{"archive_size_bytes":1557482946,"id":"190","kind":"VM","logical_id":"vm190"},{"archive_size_bytes":10998362942,"id":"9130","kind":"VM","logical_id":"vm9130"},{"archive_size_bytes":386825665,"id":"200","kind":"VM","logical_id":"vm200"},{"archive_size_bytes":3192,"id":"filebrowser","kind":"APP","logical_id":"app-filebrowser"},{"archive_size_bytes":153389,"id":"skladchik-reports-monitor","kind":"APP","logical_id":"app-skladchik-reports-monitor"},{"archive_size_bytes":270193550,"id":"immich","kind":"APP","logical_id":"app-immich"},{"archive_size_bytes":75379068,"id":"hbarhub","kind":"APP","logical_id":"app-hbarhub"},{"archive_size_bytes":8959291,"id":"control-bootstrap-v2","kind":"CONTROL","logical_id":"control-bootstrap-v2"}]}
|
|
{"clouds":[{"free_bytes":751477313023,"remote":"pve01-mail-01-crypt","total_bytes":1099511627776,"used_bytes":348034314753},{"free_bytes":751477249164,"remote":"pve01-mail-02-crypt","total_bytes":1099511627776,"used_bytes":348034378612}],"observed_at_epoch":1788491188,"schema":"homelab-backup-observability-snapshot-v1","targets":[{"archive_size_bytes":476345840,"id":"110","kind":"VM","logical_id":"vm110"},{"archive_size_bytes":369892675,"id":"111","kind":"VM","logical_id":"vm111"},{"archive_size_bytes":223672749,"id":"112","kind":"VM","logical_id":"vm112"},{"archive_size_bytes":221529138,"id":"113","kind":"VM","logical_id":"vm113"},{"archive_size_bytes":10611899437,"id":"130","kind":"VM","logical_id":"vm130"},{"archive_size_bytes":892033158,"id":"150","kind":"VM","logical_id":"vm150"},{"archive_size_bytes":3553006761,"id":"160","kind":"VM","logical_id":"vm160"},{"archive_size_bytes":7937058725,"id":"170","kind":"VM","logical_id":"vm170"},{"archive_size_bytes":2026047659,"id":"171","kind":"VM","logical_id":"vm171"},{"archive_size_bytes":1171789442,"id":"180","kind":"VM","logical_id":"vm180"},{"archive_size_bytes":1557482946,"id":"190","kind":"VM","logical_id":"vm190"},{"archive_size_bytes":10998362942,"id":"9130","kind":"VM","logical_id":"vm9130"},{"archive_size_bytes":386825665,"id":"200","kind":"VM","logical_id":"vm200"},{"archive_size_bytes":3192,"id":"filebrowser","kind":"APP","logical_id":"app-filebrowser"},{"archive_size_bytes":153389,"id":"skladchik-reports-monitor","kind":"APP","logical_id":"app-skladchik-reports-monitor"},{"archive_size_bytes":270193550,"id":"immich","kind":"APP","logical_id":"app-immich"},{"archive_size_bytes":75379068,"id":"hbarhub","kind":"APP","logical_id":"app-hbarhub"},{"archive_size_bytes":8959291,"id":"control-bootstrap-v2","kind":"CONTROL","logical_id":"control-bootstrap-v2"}]}
|
|
{"clouds":[{"free_bytes":835463704954,"remote":"pve01-mail-01-crypt","total_bytes":1099511627776,"used_bytes":264047922822},{"free_bytes":835463641095,"remote":"pve01-mail-02-crypt","total_bytes":1099511627776,"used_bytes":264047986681}],"observed_at_epoch":1788496558,"schema":"homelab-backup-observability-snapshot-v1","targets":[{"archive_size_bytes":476345840,"id":"110","kind":"VM","logical_id":"vm110"},{"archive_size_bytes":369892675,"id":"111","kind":"VM","logical_id":"vm111"},{"archive_size_bytes":223672749,"id":"112","kind":"VM","logical_id":"vm112"},{"archive_size_bytes":221529138,"id":"113","kind":"VM","logical_id":"vm113"},{"archive_size_bytes":10611899437,"id":"130","kind":"VM","logical_id":"vm130"},{"archive_size_bytes":892033158,"id":"150","kind":"VM","logical_id":"vm150"},{"archive_size_bytes":3553006761,"id":"160","kind":"VM","logical_id":"vm160"},{"archive_size_bytes":7937058725,"id":"170","kind":"VM","logical_id":"vm170"},{"archive_size_bytes":2026047659,"id":"171","kind":"VM","logical_id":"vm171"},{"archive_size_bytes":1171789442,"id":"180","kind":"VM","logical_id":"vm180"},{"archive_size_bytes":1557482946,"id":"190","kind":"VM","logical_id":"vm190"},{"archive_size_bytes":10998362942,"id":"9130","kind":"VM","logical_id":"vm9130"},{"archive_size_bytes":386825665,"id":"200","kind":"VM","logical_id":"vm200"},{"archive_size_bytes":3192,"id":"filebrowser","kind":"APP","logical_id":"app-filebrowser"},{"archive_size_bytes":153389,"id":"skladchik-reports-monitor","kind":"APP","logical_id":"app-skladchik-reports-monitor"},{"archive_size_bytes":270193550,"id":"immich","kind":"APP","logical_id":"app-immich"},{"archive_size_bytes":75379068,"id":"hbarhub","kind":"APP","logical_id":"app-hbarhub"},{"archive_size_bytes":8959291,"id":"control-bootstrap-v2","kind":"CONTROL","logical_id":"control-bootstrap-v2"}]}
|
|
{"clouds":[{"free_bytes":815114087404,"remote":"pve01-mail-01-crypt","total_bytes":1099511627776,"used_bytes":284397540372},{"free_bytes":815114023545,"remote":"pve01-mail-02-crypt","total_bytes":1099511627776,"used_bytes":284397604231}],"observed_at_epoch":1788503153,"schema":"homelab-backup-observability-snapshot-v1","targets":[{"archive_size_bytes":478357303,"id":"110","kind":"VM","logical_id":"vm110"},{"archive_size_bytes":369892675,"id":"111","kind":"VM","logical_id":"vm111"},{"archive_size_bytes":223672749,"id":"112","kind":"VM","logical_id":"vm112"},{"archive_size_bytes":221529138,"id":"113","kind":"VM","logical_id":"vm113"},{"archive_size_bytes":887338816,"id":"150","kind":"VM","logical_id":"vm150"},{"archive_size_bytes":3553006761,"id":"160","kind":"VM","logical_id":"vm160"},{"archive_size_bytes":7974616162,"id":"170","kind":"VM","logical_id":"vm170"},{"archive_size_bytes":2026047659,"id":"171","kind":"VM","logical_id":"vm171"},{"archive_size_bytes":1557482946,"id":"190","kind":"VM","logical_id":"vm190"},{"archive_size_bytes":10998362942,"id":"9130","kind":"VM","logical_id":"vm9130"},{"archive_size_bytes":3192,"id":"filebrowser","kind":"APP","logical_id":"app-filebrowser"},{"archive_size_bytes":153389,"id":"skladchik-reports-monitor","kind":"APP","logical_id":"app-skladchik-reports-monitor"},{"archive_size_bytes":270193550,"id":"immich","kind":"APP","logical_id":"app-immich"},{"archive_size_bytes":75379068,"id":"hbarhub","kind":"APP","logical_id":"app-hbarhub"},{"archive_size_bytes":8959291,"id":"control-bootstrap-v2","kind":"CONTROL","logical_id":"control-bootstrap-v2"}]}
|
|
{"clouds":[{"free_bytes":812739438805,"remote":"pve01-mail-01-crypt","total_bytes":1099511627776,"used_bytes":286772188971},{"free_bytes":812739374946,"remote":"pve01-mail-02-crypt","total_bytes":1099511627776,"used_bytes":286772252830}],"observed_at_epoch":1788507354,"schema":"homelab-backup-observability-snapshot-v1","targets":[{"archive_size_bytes":478357303,"id":"110","kind":"VM","logical_id":"vm110"},{"archive_size_bytes":369892675,"id":"111","kind":"VM","logical_id":"vm111"},{"archive_size_bytes":223672749,"id":"112","kind":"VM","logical_id":"vm112"},{"archive_size_bytes":221529138,"id":"113","kind":"VM","logical_id":"vm113"},{"archive_size_bytes":10620176786,"id":"130","kind":"VM","logical_id":"vm130"},{"archive_size_bytes":887338816,"id":"150","kind":"VM","logical_id":"vm150"},{"archive_size_bytes":3553006761,"id":"160","kind":"VM","logical_id":"vm160"},{"archive_size_bytes":7974616162,"id":"170","kind":"VM","logical_id":"vm170"},{"archive_size_bytes":2028255985,"id":"171","kind":"VM","logical_id":"vm171"},{"archive_size_bytes":1557482946,"id":"190","kind":"VM","logical_id":"vm190"},{"archive_size_bytes":10998362942,"id":"9130","kind":"VM","logical_id":"vm9130"},{"archive_size_bytes":3197,"id":"filebrowser","kind":"APP","logical_id":"app-filebrowser"},{"archive_size_bytes":154716,"id":"skladchik-reports-monitor","kind":"APP","logical_id":"app-skladchik-reports-monitor"},{"archive_size_bytes":270193492,"id":"immich","kind":"APP","logical_id":"app-immich"},{"archive_size_bytes":75453412,"id":"hbarhub","kind":"APP","logical_id":"app-hbarhub"},{"archive_size_bytes":8959291,"id":"control-bootstrap-v2","kind":"CONTROL","logical_id":"control-bootstrap-v2"}]}
|
|
{"clouds":[{"free_bytes":809167318295,"remote":"pve01-mail-01-crypt","total_bytes":1099511627776,"used_bytes":290344309481},{"free_bytes":809167254436,"remote":"pve01-mail-02-crypt","total_bytes":1099511627776,"used_bytes":290344373340}],"observed_at_epoch":1788512752,"schema":"homelab-backup-observability-snapshot-v1","targets":[{"archive_size_bytes":478357303,"id":"110","kind":"VM","logical_id":"vm110"},{"archive_size_bytes":369892675,"id":"111","kind":"VM","logical_id":"vm111"},{"archive_size_bytes":223672749,"id":"112","kind":"VM","logical_id":"vm112"},{"archive_size_bytes":221529138,"id":"113","kind":"VM","logical_id":"vm113"},{"archive_size_bytes":10620176786,"id":"130","kind":"VM","logical_id":"vm130"},{"archive_size_bytes":887338816,"id":"150","kind":"VM","logical_id":"vm150"},{"archive_size_bytes":3571246511,"id":"160","kind":"VM","logical_id":"vm160"},{"archive_size_bytes":7974616162,"id":"170","kind":"VM","logical_id":"vm170"},{"archive_size_bytes":2028255985,"id":"171","kind":"VM","logical_id":"vm171"},{"archive_size_bytes":1557482946,"id":"190","kind":"VM","logical_id":"vm190"},{"archive_size_bytes":10998362942,"id":"9130","kind":"VM","logical_id":"vm9130"},{"archive_size_bytes":3197,"id":"filebrowser","kind":"APP","logical_id":"app-filebrowser"},{"archive_size_bytes":154716,"id":"skladchik-reports-monitor","kind":"APP","logical_id":"app-skladchik-reports-monitor"},{"archive_size_bytes":270193492,"id":"immich","kind":"APP","logical_id":"app-immich"},{"archive_size_bytes":75453412,"id":"hbarhub","kind":"APP","logical_id":"app-hbarhub"},{"archive_size_bytes":8959291,"id":"control-bootstrap-v2","kind":"CONTROL","logical_id":"control-bootstrap-v2"}]}
|
|
{"clouds":[{"free_bytes":809158344080,"remote":"pve01-mail-01-crypt","total_bytes":1099511627776,"used_bytes":290353283696},{"free_bytes":809158280221,"remote":"pve01-mail-02-crypt","total_bytes":1099511627776,"used_bytes":290353347555}],"observed_at_epoch":1788516392,"schema":"homelab-backup-observability-snapshot-v1","targets":[{"archive_size_bytes":478357303,"id":"110","kind":"VM","logical_id":"vm110"},{"archive_size_bytes":369892675,"id":"111","kind":"VM","logical_id":"vm111"},{"archive_size_bytes":223672749,"id":"112","kind":"VM","logical_id":"vm112"},{"archive_size_bytes":221529138,"id":"113","kind":"VM","logical_id":"vm113"},{"archive_size_bytes":10620176786,"id":"130","kind":"VM","logical_id":"vm130"},{"archive_size_bytes":887338816,"id":"150","kind":"VM","logical_id":"vm150"},{"archive_size_bytes":3571246511,"id":"160","kind":"VM","logical_id":"vm160"},{"archive_size_bytes":7974616162,"id":"170","kind":"VM","logical_id":"vm170"},{"archive_size_bytes":2028255985,"id":"171","kind":"VM","logical_id":"vm171"},{"archive_size_bytes":1557482946,"id":"190","kind":"VM","logical_id":"vm190"},{"archive_size_bytes":10998362942,"id":"9130","kind":"VM","logical_id":"vm9130"},{"archive_size_bytes":3197,"id":"filebrowser","kind":"APP","logical_id":"app-filebrowser"},{"archive_size_bytes":154716,"id":"skladchik-reports-monitor","kind":"APP","logical_id":"app-skladchik-reports-monitor"},{"archive_size_bytes":270193492,"id":"immich","kind":"APP","logical_id":"app-immich"},{"archive_size_bytes":75453412,"id":"hbarhub","kind":"APP","logical_id":"app-hbarhub"},{"archive_size_bytes":8970684,"id":"control-bootstrap-v2","kind":"CONTROL","logical_id":"control-bootstrap-v2"}]}
|
|
{"clouds":[{"free_bytes":809158344080,"remote":"pve01-mail-01-crypt","total_bytes":1099511627776,"used_bytes":290353283696},{"free_bytes":809158280221,"remote":"pve01-mail-02-crypt","total_bytes":1099511627776,"used_bytes":290353347555}],"observed_at_epoch":1788520010,"schema":"homelab-backup-observability-snapshot-v1","targets":[{"archive_size_bytes":478357303,"id":"110","kind":"VM","logical_id":"vm110"},{"archive_size_bytes":369892675,"id":"111","kind":"VM","logical_id":"vm111"},{"archive_size_bytes":223672749,"id":"112","kind":"VM","logical_id":"vm112"},{"archive_size_bytes":221529138,"id":"113","kind":"VM","logical_id":"vm113"},{"archive_size_bytes":10620176786,"id":"130","kind":"VM","logical_id":"vm130"},{"archive_size_bytes":887338816,"id":"150","kind":"VM","logical_id":"vm150"},{"archive_size_bytes":3571246511,"id":"160","kind":"VM","logical_id":"vm160"},{"archive_size_bytes":7974616162,"id":"170","kind":"VM","logical_id":"vm170"},{"archive_size_bytes":2028255985,"id":"171","kind":"VM","logical_id":"vm171"},{"archive_size_bytes":1557482946,"id":"190","kind":"VM","logical_id":"vm190"},{"archive_size_bytes":10998362942,"id":"9130","kind":"VM","logical_id":"vm9130"},{"archive_size_bytes":3197,"id":"filebrowser","kind":"APP","logical_id":"app-filebrowser"},{"archive_size_bytes":154716,"id":"skladchik-reports-monitor","kind":"APP","logical_id":"app-skladchik-reports-monitor"},{"archive_size_bytes":270193492,"id":"immich","kind":"APP","logical_id":"app-immich"},{"archive_size_bytes":75453412,"id":"hbarhub","kind":"APP","logical_id":"app-hbarhub"},{"archive_size_bytes":8970684,"id":"control-bootstrap-v2","kind":"CONTROL","logical_id":"control-bootstrap-v2"}]}
|
|
{"clouds":[{"free_bytes":809158344080,"remote":"pve01-mail-01-crypt","total_bytes":1099511627776,"used_bytes":290353283696},{"free_bytes":809158280221,"remote":"pve01-mail-02-crypt","total_bytes":1099511627776,"used_bytes":290353347555}],"observed_at_epoch":1788525352,"schema":"homelab-backup-observability-snapshot-v1","targets":[{"archive_size_bytes":478357303,"id":"110","kind":"VM","logical_id":"vm110"},{"archive_size_bytes":369892675,"id":"111","kind":"VM","logical_id":"vm111"},{"archive_size_bytes":223672749,"id":"112","kind":"VM","logical_id":"vm112"},{"archive_size_bytes":221529138,"id":"113","kind":"VM","logical_id":"vm113"},{"archive_size_bytes":10620176786,"id":"130","kind":"VM","logical_id":"vm130"},{"archive_size_bytes":887338816,"id":"150","kind":"VM","logical_id":"vm150"},{"archive_size_bytes":3571246511,"id":"160","kind":"VM","logical_id":"vm160"},{"archive_size_bytes":7974616162,"id":"170","kind":"VM","logical_id":"vm170"},{"archive_size_bytes":2028255985,"id":"171","kind":"VM","logical_id":"vm171"},{"archive_size_bytes":1557482946,"id":"190","kind":"VM","logical_id":"vm190"},{"archive_size_bytes":10998362942,"id":"9130","kind":"VM","logical_id":"vm9130"},{"archive_size_bytes":3197,"id":"filebrowser","kind":"APP","logical_id":"app-filebrowser"},{"archive_size_bytes":154716,"id":"skladchik-reports-monitor","kind":"APP","logical_id":"app-skladchik-reports-monitor"},{"archive_size_bytes":270193492,"id":"immich","kind":"APP","logical_id":"app-immich"},{"archive_size_bytes":75453412,"id":"hbarhub","kind":"APP","logical_id":"app-hbarhub"},{"archive_size_bytes":8970684,"id":"control-bootstrap-v2","kind":"CONTROL","logical_id":"control-bootstrap-v2"}]}
|
|
{"clouds":[{"free_bytes":809158344080,"remote":"pve01-mail-01-crypt","total_bytes":1099511627776,"used_bytes":290353283696},{"free_bytes":809158280221,"remote":"pve01-mail-02-crypt","total_bytes":1099511627776,"used_bytes":290353347555}],"observed_at_epoch":1788528952,"schema":"homelab-backup-observability-snapshot-v1","targets":[{"archive_size_bytes":478357303,"id":"110","kind":"VM","logical_id":"vm110"},{"archive_size_bytes":369892675,"id":"111","kind":"VM","logical_id":"vm111"},{"archive_size_bytes":223672749,"id":"112","kind":"VM","logical_id":"vm112"},{"archive_size_bytes":221529138,"id":"113","kind":"VM","logical_id":"vm113"},{"archive_size_bytes":10620176786,"id":"130","kind":"VM","logical_id":"vm130"},{"archive_size_bytes":887338816,"id":"150","kind":"VM","logical_id":"vm150"},{"archive_size_bytes":3571246511,"id":"160","kind":"VM","logical_id":"vm160"},{"archive_size_bytes":7974616162,"id":"170","kind":"VM","logical_id":"vm170"},{"archive_size_bytes":2028255985,"id":"171","kind":"VM","logical_id":"vm171"},{"archive_size_bytes":1557482946,"id":"190","kind":"VM","logical_id":"vm190"},{"archive_size_bytes":10998362942,"id":"9130","kind":"VM","logical_id":"vm9130"},{"archive_size_bytes":3197,"id":"filebrowser","kind":"APP","logical_id":"app-filebrowser"},{"archive_size_bytes":154716,"id":"skladchik-reports-monitor","kind":"APP","logical_id":"app-skladchik-reports-monitor"},{"archive_size_bytes":270193492,"id":"immich","kind":"APP","logical_id":"app-immich"},{"archive_size_bytes":75453412,"id":"hbarhub","kind":"APP","logical_id":"app-hbarhub"},{"archive_size_bytes":8970684,"id":"control-bootstrap-v2","kind":"CONTROL","logical_id":"control-bootstrap-v2"}]}
|
|
{"clouds":[{"free_bytes":809158344080,"remote":"pve01-mail-01-crypt","total_bytes":1099511627776,"used_bytes":290353283696},{"free_bytes":809158280221,"remote":"pve01-mail-02-crypt","total_bytes":1099511627776,"used_bytes":290353347555}],"observed_at_epoch":1788532552,"schema":"homelab-backup-observability-snapshot-v1","targets":[{"archive_size_bytes":478357303,"id":"110","kind":"VM","logical_id":"vm110"},{"archive_size_bytes":369892675,"id":"111","kind":"VM","logical_id":"vm111"},{"archive_size_bytes":223672749,"id":"112","kind":"VM","logical_id":"vm112"},{"archive_size_bytes":221529138,"id":"113","kind":"VM","logical_id":"vm113"},{"archive_size_bytes":10620176786,"id":"130","kind":"VM","logical_id":"vm130"},{"archive_size_bytes":887338816,"id":"150","kind":"VM","logical_id":"vm150"},{"archive_size_bytes":3571246511,"id":"160","kind":"VM","logical_id":"vm160"},{"archive_size_bytes":7974616162,"id":"170","kind":"VM","logical_id":"vm170"},{"archive_size_bytes":2028255985,"id":"171","kind":"VM","logical_id":"vm171"},{"archive_size_bytes":1557482946,"id":"190","kind":"VM","logical_id":"vm190"},{"archive_size_bytes":10998362942,"id":"9130","kind":"VM","logical_id":"vm9130"},{"archive_size_bytes":3197,"id":"filebrowser","kind":"APP","logical_id":"app-filebrowser"},{"archive_size_bytes":154716,"id":"skladchik-reports-monitor","kind":"APP","logical_id":"app-skladchik-reports-monitor"},{"archive_size_bytes":270193492,"id":"immich","kind":"APP","logical_id":"app-immich"},{"archive_size_bytes":75453412,"id":"hbarhub","kind":"APP","logical_id":"app-hbarhub"},{"archive_size_bytes":8970684,"id":"control-bootstrap-v2","kind":"CONTROL","logical_id":"control-bootstrap-v2"}]}
|
|
{"clouds":[{"free_bytes":809158344080,"remote":"pve01-mail-01-crypt","total_bytes":1099511627776,"used_bytes":290353283696},{"free_bytes":809158280221,"remote":"pve01-mail-02-crypt","total_bytes":1099511627776,"used_bytes":290353347555}],"observed_at_epoch":1788536192,"schema":"homelab-backup-observability-snapshot-v1","targets":[{"archive_size_bytes":478357303,"id":"110","kind":"VM","logical_id":"vm110"},{"archive_size_bytes":369892675,"id":"111","kind":"VM","logical_id":"vm111"},{"archive_size_bytes":223672749,"id":"112","kind":"VM","logical_id":"vm112"},{"archive_size_bytes":221529138,"id":"113","kind":"VM","logical_id":"vm113"},{"archive_size_bytes":10620176786,"id":"130","kind":"VM","logical_id":"vm130"},{"archive_size_bytes":887338816,"id":"150","kind":"VM","logical_id":"vm150"},{"archive_size_bytes":3571246511,"id":"160","kind":"VM","logical_id":"vm160"},{"archive_size_bytes":7974616162,"id":"170","kind":"VM","logical_id":"vm170"},{"archive_size_bytes":2028255985,"id":"171","kind":"VM","logical_id":"vm171"},{"archive_size_bytes":1557482946,"id":"190","kind":"VM","logical_id":"vm190"},{"archive_size_bytes":10998362942,"id":"9130","kind":"VM","logical_id":"vm9130"},{"archive_size_bytes":3197,"id":"filebrowser","kind":"APP","logical_id":"app-filebrowser"},{"archive_size_bytes":154716,"id":"skladchik-reports-monitor","kind":"APP","logical_id":"app-skladchik-reports-monitor"},{"archive_size_bytes":270193492,"id":"immich","kind":"APP","logical_id":"app-immich"},{"archive_size_bytes":75453412,"id":"hbarhub","kind":"APP","logical_id":"app-hbarhub"},{"archive_size_bytes":8970684,"id":"control-bootstrap-v2","kind":"CONTROL","logical_id":"control-bootstrap-v2"}]}
|
|
{"clouds":[{"free_bytes":809158344080,"remote":"pve01-mail-01-crypt","total_bytes":1099511627776,"used_bytes":290353283696},{"free_bytes":809158280221,"remote":"pve01-mail-02-crypt","total_bytes":1099511627776,"used_bytes":290353347555}],"observed_at_epoch":1788541532,"schema":"homelab-backup-observability-snapshot-v1","targets":[{"archive_size_bytes":478357303,"id":"110","kind":"VM","logical_id":"vm110"},{"archive_size_bytes":369892675,"id":"111","kind":"VM","logical_id":"vm111"},{"archive_size_bytes":223672749,"id":"112","kind":"VM","logical_id":"vm112"},{"archive_size_bytes":221529138,"id":"113","kind":"VM","logical_id":"vm113"},{"archive_size_bytes":10620176786,"id":"130","kind":"VM","logical_id":"vm130"},{"archive_size_bytes":887338816,"id":"150","kind":"VM","logical_id":"vm150"},{"archive_size_bytes":3571246511,"id":"160","kind":"VM","logical_id":"vm160"},{"archive_size_bytes":7974616162,"id":"170","kind":"VM","logical_id":"vm170"},{"archive_size_bytes":2028255985,"id":"171","kind":"VM","logical_id":"vm171"},{"archive_size_bytes":1557482946,"id":"190","kind":"VM","logical_id":"vm190"},{"archive_size_bytes":10998362942,"id":"9130","kind":"VM","logical_id":"vm9130"},{"archive_size_bytes":3197,"id":"filebrowser","kind":"APP","logical_id":"app-filebrowser"},{"archive_size_bytes":154716,"id":"skladchik-reports-monitor","kind":"APP","logical_id":"app-skladchik-reports-monitor"},{"archive_size_bytes":270193492,"id":"immich","kind":"APP","logical_id":"app-immich"},{"archive_size_bytes":75453412,"id":"hbarhub","kind":"APP","logical_id":"app-hbarhub"},{"archive_size_bytes":8970684,"id":"control-bootstrap-v2","kind":"CONTROL","logical_id":"control-bootstrap-v2"}]}
|
|
{"clouds":[{"free_bytes":809158344080,"remote":"pve01-mail-01-crypt","total_bytes":1099511627776,"used_bytes":290353283696},{"free_bytes":809158280221,"remote":"pve01-mail-02-crypt","total_bytes":1099511627776,"used_bytes":290353347555}],"observed_at_epoch":1788545210,"schema":"homelab-backup-observability-snapshot-v1","targets":[{"archive_size_bytes":478357303,"id":"110","kind":"VM","logical_id":"vm110"},{"archive_size_bytes":369892675,"id":"111","kind":"VM","logical_id":"vm111"},{"archive_size_bytes":223672749,"id":"112","kind":"VM","logical_id":"vm112"},{"archive_size_bytes":221529138,"id":"113","kind":"VM","logical_id":"vm113"},{"archive_size_bytes":10620176786,"id":"130","kind":"VM","logical_id":"vm130"},{"archive_size_bytes":887338816,"id":"150","kind":"VM","logical_id":"vm150"},{"archive_size_bytes":3571246511,"id":"160","kind":"VM","logical_id":"vm160"},{"archive_size_bytes":7974616162,"id":"170","kind":"VM","logical_id":"vm170"},{"archive_size_bytes":2028255985,"id":"171","kind":"VM","logical_id":"vm171"},{"archive_size_bytes":1557482946,"id":"190","kind":"VM","logical_id":"vm190"},{"archive_size_bytes":10998362942,"id":"9130","kind":"VM","logical_id":"vm9130"},{"archive_size_bytes":3197,"id":"filebrowser","kind":"APP","logical_id":"app-filebrowser"},{"archive_size_bytes":154716,"id":"skladchik-reports-monitor","kind":"APP","logical_id":"app-skladchik-reports-monitor"},{"archive_size_bytes":270193492,"id":"immich","kind":"APP","logical_id":"app-immich"},{"archive_size_bytes":75453412,"id":"hbarhub","kind":"APP","logical_id":"app-hbarhub"},{"archive_size_bytes":8970684,"id":"control-bootstrap-v2","kind":"CONTROL","logical_id":"control-bootstrap-v2"}]}
|
|
{"clouds":[{"free_bytes":789729590488,"remote":"pve01-mail-01-crypt","total_bytes":1099511627776,"used_bytes":309782037288},{"free_bytes":789729526629,"remote":"pve01-mail-02-crypt","total_bytes":1099511627776,"used_bytes":309782101147}],"observed_at_epoch":1788550592,"schema":"homelab-backup-observability-snapshot-v1","targets":[{"archive_size_bytes":478357303,"id":"110","kind":"VM","logical_id":"vm110"},{"archive_size_bytes":369892675,"id":"111","kind":"VM","logical_id":"vm111"},{"archive_size_bytes":223672749,"id":"112","kind":"VM","logical_id":"vm112"},{"archive_size_bytes":221529138,"id":"113","kind":"VM","logical_id":"vm113"},{"archive_size_bytes":10614525791,"id":"130","kind":"VM","logical_id":"vm130"},{"archive_size_bytes":887338816,"id":"150","kind":"VM","logical_id":"vm150"},{"archive_size_bytes":3571246511,"id":"160","kind":"VM","logical_id":"vm160"},{"archive_size_bytes":6440615330,"id":"170","kind":"VM","logical_id":"vm170"},{"archive_size_bytes":1557482946,"id":"190","kind":"VM","logical_id":"vm190"},{"archive_size_bytes":10998362942,"id":"9130","kind":"VM","logical_id":"vm9130"},{"archive_size_bytes":3182,"id":"filebrowser","kind":"APP","logical_id":"app-filebrowser"},{"archive_size_bytes":156203,"id":"skladchik-reports-monitor","kind":"APP","logical_id":"app-skladchik-reports-monitor"},{"archive_size_bytes":270193546,"id":"immich","kind":"APP","logical_id":"app-immich"},{"archive_size_bytes":75522547,"id":"hbarhub","kind":"APP","logical_id":"app-hbarhub"},{"archive_size_bytes":8970684,"id":"control-bootstrap-v2","kind":"CONTROL","logical_id":"control-bootstrap-v2"}]}
|
|
{"clouds":[{"free_bytes":789729589363,"remote":"pve01-mail-01-crypt","total_bytes":1099511627776,"used_bytes":309782038413},{"free_bytes":789729525504,"remote":"pve01-mail-02-crypt","total_bytes":1099511627776,"used_bytes":309782102272}],"observed_at_epoch":1788554210,"schema":"homelab-backup-observability-snapshot-v1","targets":[{"archive_size_bytes":478357303,"id":"110","kind":"VM","logical_id":"vm110"},{"archive_size_bytes":369892675,"id":"111","kind":"VM","logical_id":"vm111"},{"archive_size_bytes":223672749,"id":"112","kind":"VM","logical_id":"vm112"},{"archive_size_bytes":221529138,"id":"113","kind":"VM","logical_id":"vm113"},{"archive_size_bytes":10614525791,"id":"130","kind":"VM","logical_id":"vm130"},{"archive_size_bytes":887338816,"id":"150","kind":"VM","logical_id":"vm150"},{"archive_size_bytes":3571246511,"id":"160","kind":"VM","logical_id":"vm160"},{"archive_size_bytes":6440615330,"id":"170","kind":"VM","logical_id":"vm170"},{"archive_size_bytes":2022982219,"id":"171","kind":"VM","logical_id":"vm171"},{"archive_size_bytes":1557482946,"id":"190","kind":"VM","logical_id":"vm190"},{"archive_size_bytes":10998362942,"id":"9130","kind":"VM","logical_id":"vm9130"},{"archive_size_bytes":3182,"id":"filebrowser","kind":"APP","logical_id":"app-filebrowser"},{"archive_size_bytes":156203,"id":"skladchik-reports-monitor","kind":"APP","logical_id":"app-skladchik-reports-monitor"},{"archive_size_bytes":270193546,"id":"immich","kind":"APP","logical_id":"app-immich"},{"archive_size_bytes":75522547,"id":"hbarhub","kind":"APP","logical_id":"app-hbarhub"},{"archive_size_bytes":8970684,"id":"control-bootstrap-v2","kind":"CONTROL","logical_id":"control-bootstrap-v2"}]}
|
|
{"clouds":[{"free_bytes":786104095977,"remote":"pve01-mail-01-crypt","total_bytes":1099511627776,"used_bytes":313407531799},{"free_bytes":786113009899,"remote":"pve01-mail-02-crypt","total_bytes":1099511627776,"used_bytes":313398617877}],"observed_at_epoch":1788559610,"schema":"homelab-backup-observability-snapshot-v1","targets":[{"archive_size_bytes":478357303,"id":"110","kind":"VM","logical_id":"vm110"},{"archive_size_bytes":369892675,"id":"111","kind":"VM","logical_id":"vm111"},{"archive_size_bytes":223672749,"id":"112","kind":"VM","logical_id":"vm112"},{"archive_size_bytes":221529138,"id":"113","kind":"VM","logical_id":"vm113"},{"archive_size_bytes":10614525791,"id":"130","kind":"VM","logical_id":"vm130"},{"archive_size_bytes":887338816,"id":"150","kind":"VM","logical_id":"vm150"},{"archive_size_bytes":3615630758,"id":"160","kind":"VM","logical_id":"vm160"},{"archive_size_bytes":6440615330,"id":"170","kind":"VM","logical_id":"vm170"},{"archive_size_bytes":2022982219,"id":"171","kind":"VM","logical_id":"vm171"},{"archive_size_bytes":1557482946,"id":"190","kind":"VM","logical_id":"vm190"},{"archive_size_bytes":10998362942,"id":"9130","kind":"VM","logical_id":"vm9130"},{"archive_size_bytes":3182,"id":"filebrowser","kind":"APP","logical_id":"app-filebrowser"},{"archive_size_bytes":156203,"id":"skladchik-reports-monitor","kind":"APP","logical_id":"app-skladchik-reports-monitor"},{"archive_size_bytes":270193546,"id":"immich","kind":"APP","logical_id":"app-immich"},{"archive_size_bytes":75522547,"id":"hbarhub","kind":"APP","logical_id":"app-hbarhub"}]}
|
|
{"clouds":[{"free_bytes":786104094818,"remote":"pve01-mail-01-crypt","total_bytes":1099511627776,"used_bytes":313407532958},{"free_bytes":786104030959,"remote":"pve01-mail-02-crypt","total_bytes":1099511627776,"used_bytes":313407596817}],"observed_at_epoch":1788564916,"schema":"homelab-backup-observability-snapshot-v1","targets":[{"archive_size_bytes":478357303,"id":"110","kind":"VM","logical_id":"vm110"},{"archive_size_bytes":369892675,"id":"111","kind":"VM","logical_id":"vm111"},{"archive_size_bytes":223672749,"id":"112","kind":"VM","logical_id":"vm112"},{"archive_size_bytes":221529138,"id":"113","kind":"VM","logical_id":"vm113"},{"archive_size_bytes":10614525791,"id":"130","kind":"VM","logical_id":"vm130"},{"archive_size_bytes":887338816,"id":"150","kind":"VM","logical_id":"vm150"},{"archive_size_bytes":3615630758,"id":"160","kind":"VM","logical_id":"vm160"},{"archive_size_bytes":6440615330,"id":"170","kind":"VM","logical_id":"vm170"},{"archive_size_bytes":2022982219,"id":"171","kind":"VM","logical_id":"vm171"},{"archive_size_bytes":1557482946,"id":"190","kind":"VM","logical_id":"vm190"},{"archive_size_bytes":10998362942,"id":"9130","kind":"VM","logical_id":"vm9130"},{"archive_size_bytes":3182,"id":"filebrowser","kind":"APP","logical_id":"app-filebrowser"},{"archive_size_bytes":156203,"id":"skladchik-reports-monitor","kind":"APP","logical_id":"app-skladchik-reports-monitor"},{"archive_size_bytes":270193546,"id":"immich","kind":"APP","logical_id":"app-immich"},{"archive_size_bytes":75522547,"id":"hbarhub","kind":"APP","logical_id":"app-hbarhub"},{"archive_size_bytes":8975409,"id":"control-bootstrap-v2","kind":"CONTROL","logical_id":"control-bootstrap-v2"}]}
|
|
{"clouds":[{"free_bytes":786104094818,"remote":"pve01-mail-01-crypt","total_bytes":1099511627776,"used_bytes":313407532958},{"free_bytes":786104030959,"remote":"pve01-mail-02-crypt","total_bytes":1099511627776,"used_bytes":313407596817}],"observed_at_epoch":1788568552,"schema":"homelab-backup-observability-snapshot-v1","targets":[{"archive_size_bytes":478357303,"id":"110","kind":"VM","logical_id":"vm110"},{"archive_size_bytes":369892675,"id":"111","kind":"VM","logical_id":"vm111"},{"archive_size_bytes":223672749,"id":"112","kind":"VM","logical_id":"vm112"},{"archive_size_bytes":221529138,"id":"113","kind":"VM","logical_id":"vm113"},{"archive_size_bytes":10614525791,"id":"130","kind":"VM","logical_id":"vm130"},{"archive_size_bytes":887338816,"id":"150","kind":"VM","logical_id":"vm150"},{"archive_size_bytes":3615630758,"id":"160","kind":"VM","logical_id":"vm160"},{"archive_size_bytes":6440615330,"id":"170","kind":"VM","logical_id":"vm170"},{"archive_size_bytes":2022982219,"id":"171","kind":"VM","logical_id":"vm171"},{"archive_size_bytes":1557482946,"id":"190","kind":"VM","logical_id":"vm190"},{"archive_size_bytes":10998362942,"id":"9130","kind":"VM","logical_id":"vm9130"},{"archive_size_bytes":3182,"id":"filebrowser","kind":"APP","logical_id":"app-filebrowser"},{"archive_size_bytes":156203,"id":"skladchik-reports-monitor","kind":"APP","logical_id":"app-skladchik-reports-monitor"},{"archive_size_bytes":270193546,"id":"immich","kind":"APP","logical_id":"app-immich"},{"archive_size_bytes":75522547,"id":"hbarhub","kind":"APP","logical_id":"app-hbarhub"},{"archive_size_bytes":8975409,"id":"control-bootstrap-v2","kind":"CONTROL","logical_id":"control-bootstrap-v2"}]}
|
|
{"clouds":[{"free_bytes":786104094818,"remote":"pve01-mail-01-crypt","total_bytes":1099511627776,"used_bytes":313407532958},{"free_bytes":786104030959,"remote":"pve01-mail-02-crypt","total_bytes":1099511627776,"used_bytes":313407596817}],"observed_at_epoch":1788572153,"schema":"homelab-backup-observability-snapshot-v1","targets":[{"archive_size_bytes":478357303,"id":"110","kind":"VM","logical_id":"vm110"},{"archive_size_bytes":369892675,"id":"111","kind":"VM","logical_id":"vm111"},{"archive_size_bytes":223672749,"id":"112","kind":"VM","logical_id":"vm112"},{"archive_size_bytes":221529138,"id":"113","kind":"VM","logical_id":"vm113"},{"archive_size_bytes":10614525791,"id":"130","kind":"VM","logical_id":"vm130"},{"archive_size_bytes":887338816,"id":"150","kind":"VM","logical_id":"vm150"},{"archive_size_bytes":3615630758,"id":"160","kind":"VM","logical_id":"vm160"},{"archive_size_bytes":6440615330,"id":"170","kind":"VM","logical_id":"vm170"},{"archive_size_bytes":2022982219,"id":"171","kind":"VM","logical_id":"vm171"},{"archive_size_bytes":1557482946,"id":"190","kind":"VM","logical_id":"vm190"},{"archive_size_bytes":10998362942,"id":"9130","kind":"VM","logical_id":"vm9130"},{"archive_size_bytes":3182,"id":"filebrowser","kind":"APP","logical_id":"app-filebrowser"},{"archive_size_bytes":156203,"id":"skladchik-reports-monitor","kind":"APP","logical_id":"app-skladchik-reports-monitor"},{"archive_size_bytes":270193546,"id":"immich","kind":"APP","logical_id":"app-immich"},{"archive_size_bytes":75522547,"id":"hbarhub","kind":"APP","logical_id":"app-hbarhub"},{"archive_size_bytes":8975409,"id":"control-bootstrap-v2","kind":"CONTROL","logical_id":"control-bootstrap-v2"}]}
|
|
{"clouds":[{"free_bytes":786104094818,"remote":"pve01-mail-01-crypt","total_bytes":1099511627776,"used_bytes":313407532958},{"free_bytes":786104030959,"remote":"pve01-mail-02-crypt","total_bytes":1099511627776,"used_bytes":313407596817}],"observed_at_epoch":1788575770,"schema":"homelab-backup-observability-snapshot-v1","targets":[{"archive_size_bytes":478357303,"id":"110","kind":"VM","logical_id":"vm110"},{"archive_size_bytes":369892675,"id":"111","kind":"VM","logical_id":"vm111"},{"archive_size_bytes":223672749,"id":"112","kind":"VM","logical_id":"vm112"},{"archive_size_bytes":221529138,"id":"113","kind":"VM","logical_id":"vm113"},{"archive_size_bytes":10614525791,"id":"130","kind":"VM","logical_id":"vm130"},{"archive_size_bytes":887338816,"id":"150","kind":"VM","logical_id":"vm150"},{"archive_size_bytes":3615630758,"id":"160","kind":"VM","logical_id":"vm160"},{"archive_size_bytes":6440615330,"id":"170","kind":"VM","logical_id":"vm170"},{"archive_size_bytes":2022982219,"id":"171","kind":"VM","logical_id":"vm171"},{"archive_size_bytes":1557482946,"id":"190","kind":"VM","logical_id":"vm190"},{"archive_size_bytes":10998362942,"id":"9130","kind":"VM","logical_id":"vm9130"},{"archive_size_bytes":3182,"id":"filebrowser","kind":"APP","logical_id":"app-filebrowser"},{"archive_size_bytes":156203,"id":"skladchik-reports-monitor","kind":"APP","logical_id":"app-skladchik-reports-monitor"},{"archive_size_bytes":270193546,"id":"immich","kind":"APP","logical_id":"app-immich"},{"archive_size_bytes":75522547,"id":"hbarhub","kind":"APP","logical_id":"app-hbarhub"},{"archive_size_bytes":8975409,"id":"control-bootstrap-v2","kind":"CONTROL","logical_id":"control-bootstrap-v2"}]}
|
|
{"clouds":[{"free_bytes":786104094818,"remote":"pve01-mail-01-crypt","total_bytes":1099511627776,"used_bytes":313407532958},{"free_bytes":786104030959,"remote":"pve01-mail-02-crypt","total_bytes":1099511627776,"used_bytes":313407596817}],"observed_at_epoch":1788579413,"schema":"homelab-backup-observability-snapshot-v1","targets":[{"archive_size_bytes":478357303,"id":"110","kind":"VM","logical_id":"vm110"},{"archive_size_bytes":369892675,"id":"111","kind":"VM","logical_id":"vm111"},{"archive_size_bytes":223672749,"id":"112","kind":"VM","logical_id":"vm112"},{"archive_size_bytes":221529138,"id":"113","kind":"VM","logical_id":"vm113"},{"archive_size_bytes":10614525791,"id":"130","kind":"VM","logical_id":"vm130"},{"archive_size_bytes":887338816,"id":"150","kind":"VM","logical_id":"vm150"},{"archive_size_bytes":3615630758,"id":"160","kind":"VM","logical_id":"vm160"},{"archive_size_bytes":6440615330,"id":"170","kind":"VM","logical_id":"vm170"},{"archive_size_bytes":2022982219,"id":"171","kind":"VM","logical_id":"vm171"},{"archive_size_bytes":1557482946,"id":"190","kind":"VM","logical_id":"vm190"},{"archive_size_bytes":10998362942,"id":"9130","kind":"VM","logical_id":"vm9130"},{"archive_size_bytes":3182,"id":"filebrowser","kind":"APP","logical_id":"app-filebrowser"},{"archive_size_bytes":156203,"id":"skladchik-reports-monitor","kind":"APP","logical_id":"app-skladchik-reports-monitor"},{"archive_size_bytes":270193546,"id":"immich","kind":"APP","logical_id":"app-immich"},{"archive_size_bytes":75522547,"id":"hbarhub","kind":"APP","logical_id":"app-hbarhub"},{"archive_size_bytes":8975409,"id":"control-bootstrap-v2","kind":"CONTROL","logical_id":"control-bootstrap-v2"}]}
|
|
{"clouds":[{"free_bytes":786104094818,"remote":"pve01-mail-01-crypt","total_bytes":1099511627776,"used_bytes":313407532958},{"free_bytes":786104030959,"remote":"pve01-mail-02-crypt","total_bytes":1099511627776,"used_bytes":313407596817}],"observed_at_epoch":1788583036,"schema":"homelab-backup-observability-snapshot-v1","targets":[{"archive_size_bytes":478357303,"id":"110","kind":"VM","logical_id":"vm110"},{"archive_size_bytes":369892675,"id":"111","kind":"VM","logical_id":"vm111"},{"archive_size_bytes":223672749,"id":"112","kind":"VM","logical_id":"vm112"},{"archive_size_bytes":221529138,"id":"113","kind":"VM","logical_id":"vm113"},{"archive_size_bytes":10614525791,"id":"130","kind":"VM","logical_id":"vm130"},{"archive_size_bytes":887338816,"id":"150","kind":"VM","logical_id":"vm150"},{"archive_size_bytes":3615630758,"id":"160","kind":"VM","logical_id":"vm160"},{"archive_size_bytes":6440615330,"id":"170","kind":"VM","logical_id":"vm170"},{"archive_size_bytes":2022982219,"id":"171","kind":"VM","logical_id":"vm171"},{"archive_size_bytes":1557482946,"id":"190","kind":"VM","logical_id":"vm190"},{"archive_size_bytes":10998362942,"id":"9130","kind":"VM","logical_id":"vm9130"},{"archive_size_bytes":3182,"id":"filebrowser","kind":"APP","logical_id":"app-filebrowser"},{"archive_size_bytes":156203,"id":"skladchik-reports-monitor","kind":"APP","logical_id":"app-skladchik-reports-monitor"},{"archive_size_bytes":270193546,"id":"immich","kind":"APP","logical_id":"app-immich"},{"archive_size_bytes":75522547,"id":"hbarhub","kind":"APP","logical_id":"app-hbarhub"},{"archive_size_bytes":8975409,"id":"control-bootstrap-v2","kind":"CONTROL","logical_id":"control-bootstrap-v2"}]}
|
|
{"clouds":[{"free_bytes":786104095252,"remote":"pve01-mail-01-crypt","total_bytes":1099511627776,"used_bytes":313407532524},{"free_bytes":786104031393,"remote":"pve01-mail-02-crypt","total_bytes":1099511627776,"used_bytes":313407596383}],"observed_at_epoch":1788588332,"schema":"homelab-backup-observability-snapshot-v1","targets":[{"archive_size_bytes":478357303,"id":"110","kind":"VM","logical_id":"vm110"},{"archive_size_bytes":369892675,"id":"111","kind":"VM","logical_id":"vm111"},{"archive_size_bytes":223672749,"id":"112","kind":"VM","logical_id":"vm112"},{"archive_size_bytes":221529138,"id":"113","kind":"VM","logical_id":"vm113"},{"archive_size_bytes":10614525791,"id":"130","kind":"VM","logical_id":"vm130"},{"archive_size_bytes":887338816,"id":"150","kind":"VM","logical_id":"vm150"},{"archive_size_bytes":3615630758,"id":"160","kind":"VM","logical_id":"vm160"},{"archive_size_bytes":6440615330,"id":"170","kind":"VM","logical_id":"vm170"},{"archive_size_bytes":2022982219,"id":"171","kind":"VM","logical_id":"vm171"},{"archive_size_bytes":1557482946,"id":"190","kind":"VM","logical_id":"vm190"},{"archive_size_bytes":10998362942,"id":"9130","kind":"VM","logical_id":"vm9130"},{"archive_size_bytes":3182,"id":"filebrowser","kind":"APP","logical_id":"app-filebrowser"},{"archive_size_bytes":156203,"id":"skladchik-reports-monitor","kind":"APP","logical_id":"app-skladchik-reports-monitor"},{"archive_size_bytes":270193546,"id":"immich","kind":"APP","logical_id":"app-immich"},{"archive_size_bytes":75522547,"id":"hbarhub","kind":"APP","logical_id":"app-hbarhub"},{"archive_size_bytes":8975409,"id":"control-bootstrap-v2","kind":"CONTROL","logical_id":"control-bootstrap-v2"}]}
|
|
{"clouds":[{"free_bytes":786104095252,"remote":"pve01-mail-01-crypt","total_bytes":1099511627776,"used_bytes":313407532524},{"free_bytes":786104031393,"remote":"pve01-mail-02-crypt","total_bytes":1099511627776,"used_bytes":313407596383}],"observed_at_epoch":1788591953,"schema":"homelab-backup-observability-snapshot-v1","targets":[{"archive_size_bytes":478357303,"id":"110","kind":"VM","logical_id":"vm110"},{"archive_size_bytes":369892675,"id":"111","kind":"VM","logical_id":"vm111"},{"archive_size_bytes":223672749,"id":"112","kind":"VM","logical_id":"vm112"},{"archive_size_bytes":221529138,"id":"113","kind":"VM","logical_id":"vm113"},{"archive_size_bytes":10614525791,"id":"130","kind":"VM","logical_id":"vm130"},{"archive_size_bytes":887338816,"id":"150","kind":"VM","logical_id":"vm150"},{"archive_size_bytes":3615630758,"id":"160","kind":"VM","logical_id":"vm160"},{"archive_size_bytes":6440615330,"id":"170","kind":"VM","logical_id":"vm170"},{"archive_size_bytes":2022982219,"id":"171","kind":"VM","logical_id":"vm171"},{"archive_size_bytes":1557482946,"id":"190","kind":"VM","logical_id":"vm190"},{"archive_size_bytes":10998362942,"id":"9130","kind":"VM","logical_id":"vm9130"},{"archive_size_bytes":3182,"id":"filebrowser","kind":"APP","logical_id":"app-filebrowser"},{"archive_size_bytes":156203,"id":"skladchik-reports-monitor","kind":"APP","logical_id":"app-skladchik-reports-monitor"},{"archive_size_bytes":270193546,"id":"immich","kind":"APP","logical_id":"app-immich"},{"archive_size_bytes":75522547,"id":"hbarhub","kind":"APP","logical_id":"app-hbarhub"},{"archive_size_bytes":8975409,"id":"control-bootstrap-v2","kind":"CONTROL","logical_id":"control-bootstrap-v2"}]}
|
|
{"clouds":[{"free_bytes":786104095252,"remote":"pve01-mail-01-crypt","total_bytes":1099511627776,"used_bytes":313407532524},{"free_bytes":786104031393,"remote":"pve01-mail-02-crypt","total_bytes":1099511627776,"used_bytes":313407596383}],"observed_at_epoch":1788595592,"schema":"homelab-backup-observability-snapshot-v1","targets":[{"archive_size_bytes":478357303,"id":"110","kind":"VM","logical_id":"vm110"},{"archive_size_bytes":369892675,"id":"111","kind":"VM","logical_id":"vm111"},{"archive_size_bytes":223672749,"id":"112","kind":"VM","logical_id":"vm112"},{"archive_size_bytes":221529138,"id":"113","kind":"VM","logical_id":"vm113"},{"archive_size_bytes":10614525791,"id":"130","kind":"VM","logical_id":"vm130"},{"archive_size_bytes":887338816,"id":"150","kind":"VM","logical_id":"vm150"},{"archive_size_bytes":3615630758,"id":"160","kind":"VM","logical_id":"vm160"},{"archive_size_bytes":6440615330,"id":"170","kind":"VM","logical_id":"vm170"},{"archive_size_bytes":2022982219,"id":"171","kind":"VM","logical_id":"vm171"},{"archive_size_bytes":1557482946,"id":"190","kind":"VM","logical_id":"vm190"},{"archive_size_bytes":10998362942,"id":"9130","kind":"VM","logical_id":"vm9130"},{"archive_size_bytes":3182,"id":"filebrowser","kind":"APP","logical_id":"app-filebrowser"},{"archive_size_bytes":156203,"id":"skladchik-reports-monitor","kind":"APP","logical_id":"app-skladchik-reports-monitor"},{"archive_size_bytes":270193546,"id":"immich","kind":"APP","logical_id":"app-immich"},{"archive_size_bytes":75522547,"id":"hbarhub","kind":"APP","logical_id":"app-hbarhub"},{"archive_size_bytes":8975409,"id":"control-bootstrap-v2","kind":"CONTROL","logical_id":"control-bootstrap-v2"}]}
|
|
{"clouds":[{"free_bytes":767079957393,"remote":"pve01-mail-01-crypt","total_bytes":1099511627776,"used_bytes":332431670383},{"free_bytes":768690899582,"remote":"pve01-mail-02-crypt","total_bytes":1099511627776,"used_bytes":330820728194}],"observed_at_epoch":1788600970,"schema":"homelab-backup-observability-snapshot-v1","targets":[{"archive_size_bytes":478357303,"id":"110","kind":"VM","logical_id":"vm110"},{"archive_size_bytes":369892675,"id":"111","kind":"VM","logical_id":"vm111"},{"archive_size_bytes":223672749,"id":"112","kind":"VM","logical_id":"vm112"},{"archive_size_bytes":221529138,"id":"113","kind":"VM","logical_id":"vm113"},{"archive_size_bytes":10591109796,"id":"130","kind":"VM","logical_id":"vm130"},{"archive_size_bytes":887338816,"id":"150","kind":"VM","logical_id":"vm150"},{"archive_size_bytes":3615630758,"id":"160","kind":"VM","logical_id":"vm160"},{"archive_size_bytes":6471807402,"id":"170","kind":"VM","logical_id":"vm170"},{"archive_size_bytes":1557482946,"id":"190","kind":"VM","logical_id":"vm190"},{"archive_size_bytes":10998362942,"id":"9130","kind":"VM","logical_id":"vm9130"},{"archive_size_bytes":3189,"id":"filebrowser","kind":"APP","logical_id":"app-filebrowser"},{"archive_size_bytes":156634,"id":"skladchik-reports-monitor","kind":"APP","logical_id":"app-skladchik-reports-monitor"},{"archive_size_bytes":270193548,"id":"immich","kind":"APP","logical_id":"app-immich"},{"archive_size_bytes":75598903,"id":"hbarhub","kind":"APP","logical_id":"app-hbarhub"},{"archive_size_bytes":8975409,"id":"control-bootstrap-v2","kind":"CONTROL","logical_id":"control-bootstrap-v2"}]}
|
|
{"clouds":[{"free_bytes":763045869314,"remote":"pve01-mail-01-crypt","total_bytes":1099511627776,"used_bytes":336465758462},{"free_bytes":763045805455,"remote":"pve01-mail-02-crypt","total_bytes":1099511627776,"used_bytes":336465822321}],"observed_at_epoch":1788604613,"schema":"homelab-backup-observability-snapshot-v1","targets":[{"archive_size_bytes":478357303,"id":"110","kind":"VM","logical_id":"vm110"},{"archive_size_bytes":369892675,"id":"111","kind":"VM","logical_id":"vm111"},{"archive_size_bytes":223672749,"id":"112","kind":"VM","logical_id":"vm112"},{"archive_size_bytes":221529138,"id":"113","kind":"VM","logical_id":"vm113"},{"archive_size_bytes":10591109796,"id":"130","kind":"VM","logical_id":"vm130"},{"archive_size_bytes":887338816,"id":"150","kind":"VM","logical_id":"vm150"},{"archive_size_bytes":6471807402,"id":"170","kind":"VM","logical_id":"vm170"},{"archive_size_bytes":2023943350,"id":"171","kind":"VM","logical_id":"vm171"},{"archive_size_bytes":1557482946,"id":"190","kind":"VM","logical_id":"vm190"},{"archive_size_bytes":10998362942,"id":"9130","kind":"VM","logical_id":"vm9130"},{"archive_size_bytes":3189,"id":"filebrowser","kind":"APP","logical_id":"app-filebrowser"},{"archive_size_bytes":156634,"id":"skladchik-reports-monitor","kind":"APP","logical_id":"app-skladchik-reports-monitor"},{"archive_size_bytes":270193548,"id":"immich","kind":"APP","logical_id":"app-immich"},{"archive_size_bytes":75598903,"id":"hbarhub","kind":"APP","logical_id":"app-hbarhub"},{"archive_size_bytes":8975409,"id":"control-bootstrap-v2","kind":"CONTROL","logical_id":"control-bootstrap-v2"}]}
|
|
{"clouds":[{"free_bytes":763036886053,"remote":"pve01-mail-01-crypt","total_bytes":1099511627776,"used_bytes":336474741723},{"free_bytes":763036822194,"remote":"pve01-mail-02-crypt","total_bytes":1099511627776,"used_bytes":336474805582}],"observed_at_epoch":1788609912,"schema":"homelab-backup-observability-snapshot-v1","targets":[{"archive_size_bytes":478357303,"id":"110","kind":"VM","logical_id":"vm110"},{"archive_size_bytes":369892675,"id":"111","kind":"VM","logical_id":"vm111"},{"archive_size_bytes":223672749,"id":"112","kind":"VM","logical_id":"vm112"},{"archive_size_bytes":221529138,"id":"113","kind":"VM","logical_id":"vm113"},{"archive_size_bytes":10591109796,"id":"130","kind":"VM","logical_id":"vm130"},{"archive_size_bytes":887338816,"id":"150","kind":"VM","logical_id":"vm150"},{"archive_size_bytes":3619770210,"id":"160","kind":"VM","logical_id":"vm160"},{"archive_size_bytes":6471807402,"id":"170","kind":"VM","logical_id":"vm170"},{"archive_size_bytes":2023943350,"id":"171","kind":"VM","logical_id":"vm171"},{"archive_size_bytes":1557482946,"id":"190","kind":"VM","logical_id":"vm190"},{"archive_size_bytes":10998362942,"id":"9130","kind":"VM","logical_id":"vm9130"},{"archive_size_bytes":3189,"id":"filebrowser","kind":"APP","logical_id":"app-filebrowser"},{"archive_size_bytes":156634,"id":"skladchik-reports-monitor","kind":"APP","logical_id":"app-skladchik-reports-monitor"},{"archive_size_bytes":270193548,"id":"immich","kind":"APP","logical_id":"app-immich"},{"archive_size_bytes":75598903,"id":"hbarhub","kind":"APP","logical_id":"app-hbarhub"},{"archive_size_bytes":8978589,"id":"control-bootstrap-v2","kind":"CONTROL","logical_id":"control-bootstrap-v2"}]}
|
|
{"clouds":[{"free_bytes":763036886053,"remote":"pve01-mail-01-crypt","total_bytes":1099511627776,"used_bytes":336474741723},{"free_bytes":763036822194,"remote":"pve01-mail-02-crypt","total_bytes":1099511627776,"used_bytes":336474805582}],"observed_at_epoch":1788613592,"schema":"homelab-backup-observability-snapshot-v1","targets":[{"archive_size_bytes":478357303,"id":"110","kind":"VM","logical_id":"vm110"},{"archive_size_bytes":369892675,"id":"111","kind":"VM","logical_id":"vm111"},{"archive_size_bytes":223672749,"id":"112","kind":"VM","logical_id":"vm112"},{"archive_size_bytes":221529138,"id":"113","kind":"VM","logical_id":"vm113"},{"archive_size_bytes":10591109796,"id":"130","kind":"VM","logical_id":"vm130"},{"archive_size_bytes":887338816,"id":"150","kind":"VM","logical_id":"vm150"},{"archive_size_bytes":3619770210,"id":"160","kind":"VM","logical_id":"vm160"},{"archive_size_bytes":6471807402,"id":"170","kind":"VM","logical_id":"vm170"},{"archive_size_bytes":2023943350,"id":"171","kind":"VM","logical_id":"vm171"},{"archive_size_bytes":1557482946,"id":"190","kind":"VM","logical_id":"vm190"},{"archive_size_bytes":10998362942,"id":"9130","kind":"VM","logical_id":"vm9130"},{"archive_size_bytes":3189,"id":"filebrowser","kind":"APP","logical_id":"app-filebrowser"},{"archive_size_bytes":156634,"id":"skladchik-reports-monitor","kind":"APP","logical_id":"app-skladchik-reports-monitor"},{"archive_size_bytes":270193548,"id":"immich","kind":"APP","logical_id":"app-immich"},{"archive_size_bytes":75598903,"id":"hbarhub","kind":"APP","logical_id":"app-hbarhub"},{"archive_size_bytes":8978589,"id":"control-bootstrap-v2","kind":"CONTROL","logical_id":"control-bootstrap-v2"}]}
|
|
{"clouds":[{"free_bytes":763036886053,"remote":"pve01-mail-01-crypt","total_bytes":1099511627776,"used_bytes":336474741723},{"free_bytes":763036822194,"remote":"pve01-mail-02-crypt","total_bytes":1099511627776,"used_bytes":336474805582}],"observed_at_epoch":1788618970,"schema":"homelab-backup-observability-snapshot-v1","targets":[{"archive_size_bytes":478357303,"id":"110","kind":"VM","logical_id":"vm110"},{"archive_size_bytes":369892675,"id":"111","kind":"VM","logical_id":"vm111"},{"archive_size_bytes":223672749,"id":"112","kind":"VM","logical_id":"vm112"},{"archive_size_bytes":221529138,"id":"113","kind":"VM","logical_id":"vm113"},{"archive_size_bytes":10591109796,"id":"130","kind":"VM","logical_id":"vm130"},{"archive_size_bytes":887338816,"id":"150","kind":"VM","logical_id":"vm150"},{"archive_size_bytes":3619770210,"id":"160","kind":"VM","logical_id":"vm160"},{"archive_size_bytes":6471807402,"id":"170","kind":"VM","logical_id":"vm170"},{"archive_size_bytes":2023943350,"id":"171","kind":"VM","logical_id":"vm171"},{"archive_size_bytes":1557482946,"id":"190","kind":"VM","logical_id":"vm190"},{"archive_size_bytes":10998362942,"id":"9130","kind":"VM","logical_id":"vm9130"},{"archive_size_bytes":3189,"id":"filebrowser","kind":"APP","logical_id":"app-filebrowser"},{"archive_size_bytes":156634,"id":"skladchik-reports-monitor","kind":"APP","logical_id":"app-skladchik-reports-monitor"},{"archive_size_bytes":270193548,"id":"immich","kind":"APP","logical_id":"app-immich"},{"archive_size_bytes":75598903,"id":"hbarhub","kind":"APP","logical_id":"app-hbarhub"},{"archive_size_bytes":8978589,"id":"control-bootstrap-v2","kind":"CONTROL","logical_id":"control-bootstrap-v2"}]}
|
|
{"clouds":[{"free_bytes":763036886053,"remote":"pve01-mail-01-crypt","total_bytes":1099511627776,"used_bytes":336474741723},{"free_bytes":763036822194,"remote":"pve01-mail-02-crypt","total_bytes":1099511627776,"used_bytes":336474805582}],"observed_at_epoch":1788624358,"schema":"homelab-backup-observability-snapshot-v1","targets":[{"archive_size_bytes":478357303,"id":"110","kind":"VM","logical_id":"vm110"},{"archive_size_bytes":369892675,"id":"111","kind":"VM","logical_id":"vm111"},{"archive_size_bytes":223672749,"id":"112","kind":"VM","logical_id":"vm112"},{"archive_size_bytes":221529138,"id":"113","kind":"VM","logical_id":"vm113"},{"archive_size_bytes":10591109796,"id":"130","kind":"VM","logical_id":"vm130"},{"archive_size_bytes":887338816,"id":"150","kind":"VM","logical_id":"vm150"},{"archive_size_bytes":3619770210,"id":"160","kind":"VM","logical_id":"vm160"},{"archive_size_bytes":6471807402,"id":"170","kind":"VM","logical_id":"vm170"},{"archive_size_bytes":2023943350,"id":"171","kind":"VM","logical_id":"vm171"},{"archive_size_bytes":1557482946,"id":"190","kind":"VM","logical_id":"vm190"},{"archive_size_bytes":10998362942,"id":"9130","kind":"VM","logical_id":"vm9130"},{"archive_size_bytes":3189,"id":"filebrowser","kind":"APP","logical_id":"app-filebrowser"},{"archive_size_bytes":156634,"id":"skladchik-reports-monitor","kind":"APP","logical_id":"app-skladchik-reports-monitor"},{"archive_size_bytes":270193548,"id":"immich","kind":"APP","logical_id":"app-immich"},{"archive_size_bytes":75598903,"id":"hbarhub","kind":"APP","logical_id":"app-hbarhub"},{"archive_size_bytes":8978589,"id":"control-bootstrap-v2","kind":"CONTROL","logical_id":"control-bootstrap-v2"}]}
|
|
{"clouds":[{"free_bytes":763036886053,"remote":"pve01-mail-01-crypt","total_bytes":1099511627776,"used_bytes":336474741723},{"free_bytes":763036822194,"remote":"pve01-mail-02-crypt","total_bytes":1099511627776,"used_bytes":336474805582}],"observed_at_epoch":1788628013,"schema":"homelab-backup-observability-snapshot-v1","targets":[{"archive_size_bytes":478357303,"id":"110","kind":"VM","logical_id":"vm110"},{"archive_size_bytes":369892675,"id":"111","kind":"VM","logical_id":"vm111"},{"archive_size_bytes":223672749,"id":"112","kind":"VM","logical_id":"vm112"},{"archive_size_bytes":221529138,"id":"113","kind":"VM","logical_id":"vm113"},{"archive_size_bytes":10591109796,"id":"130","kind":"VM","logical_id":"vm130"},{"archive_size_bytes":887338816,"id":"150","kind":"VM","logical_id":"vm150"},{"archive_size_bytes":3619770210,"id":"160","kind":"VM","logical_id":"vm160"},{"archive_size_bytes":6471807402,"id":"170","kind":"VM","logical_id":"vm170"},{"archive_size_bytes":2023943350,"id":"171","kind":"VM","logical_id":"vm171"},{"archive_size_bytes":1557482946,"id":"190","kind":"VM","logical_id":"vm190"},{"archive_size_bytes":10998362942,"id":"9130","kind":"VM","logical_id":"vm9130"},{"archive_size_bytes":3189,"id":"filebrowser","kind":"APP","logical_id":"app-filebrowser"},{"archive_size_bytes":156634,"id":"skladchik-reports-monitor","kind":"APP","logical_id":"app-skladchik-reports-monitor"},{"archive_size_bytes":270193548,"id":"immich","kind":"APP","logical_id":"app-immich"},{"archive_size_bytes":75598903,"id":"hbarhub","kind":"APP","logical_id":"app-hbarhub"},{"archive_size_bytes":8978589,"id":"control-bootstrap-v2","kind":"CONTROL","logical_id":"control-bootstrap-v2"}]}
|
|
{"clouds":[{"free_bytes":763036886053,"remote":"pve01-mail-01-crypt","total_bytes":1099511627776,"used_bytes":336474741723},{"free_bytes":763036822194,"remote":"pve01-mail-02-crypt","total_bytes":1099511627776,"used_bytes":336474805582}],"observed_at_epoch":1788633392,"schema":"homelab-backup-observability-snapshot-v1","targets":[{"archive_size_bytes":478357303,"id":"110","kind":"VM","logical_id":"vm110"},{"archive_size_bytes":369892675,"id":"111","kind":"VM","logical_id":"vm111"},{"archive_size_bytes":223672749,"id":"112","kind":"VM","logical_id":"vm112"},{"archive_size_bytes":221529138,"id":"113","kind":"VM","logical_id":"vm113"},{"archive_size_bytes":10591109796,"id":"130","kind":"VM","logical_id":"vm130"},{"archive_size_bytes":887338816,"id":"150","kind":"VM","logical_id":"vm150"},{"archive_size_bytes":3619770210,"id":"160","kind":"VM","logical_id":"vm160"},{"archive_size_bytes":6471807402,"id":"170","kind":"VM","logical_id":"vm170"},{"archive_size_bytes":2023943350,"id":"171","kind":"VM","logical_id":"vm171"},{"archive_size_bytes":1557482946,"id":"190","kind":"VM","logical_id":"vm190"},{"archive_size_bytes":10998362942,"id":"9130","kind":"VM","logical_id":"vm9130"},{"archive_size_bytes":3189,"id":"filebrowser","kind":"APP","logical_id":"app-filebrowser"},{"archive_size_bytes":156634,"id":"skladchik-reports-monitor","kind":"APP","logical_id":"app-skladchik-reports-monitor"},{"archive_size_bytes":270193548,"id":"immich","kind":"APP","logical_id":"app-immich"},{"archive_size_bytes":75598903,"id":"hbarhub","kind":"APP","logical_id":"app-hbarhub"},{"archive_size_bytes":8978589,"id":"control-bootstrap-v2","kind":"CONTROL","logical_id":"control-bootstrap-v2"}]}
|
|
{"clouds":[{"free_bytes":763036886053,"remote":"pve01-mail-01-crypt","total_bytes":1099511627776,"used_bytes":336474741723},{"free_bytes":763036822194,"remote":"pve01-mail-02-crypt","total_bytes":1099511627776,"used_bytes":336474805582}],"observed_at_epoch":1788638772,"schema":"homelab-backup-observability-snapshot-v1","targets":[{"archive_size_bytes":478357303,"id":"110","kind":"VM","logical_id":"vm110"},{"archive_size_bytes":369892675,"id":"111","kind":"VM","logical_id":"vm111"},{"archive_size_bytes":223672749,"id":"112","kind":"VM","logical_id":"vm112"},{"archive_size_bytes":221529138,"id":"113","kind":"VM","logical_id":"vm113"},{"archive_size_bytes":10591109796,"id":"130","kind":"VM","logical_id":"vm130"},{"archive_size_bytes":887338816,"id":"150","kind":"VM","logical_id":"vm150"},{"archive_size_bytes":3619770210,"id":"160","kind":"VM","logical_id":"vm160"},{"archive_size_bytes":6471807402,"id":"170","kind":"VM","logical_id":"vm170"},{"archive_size_bytes":2023943350,"id":"171","kind":"VM","logical_id":"vm171"},{"archive_size_bytes":1557482946,"id":"190","kind":"VM","logical_id":"vm190"},{"archive_size_bytes":10998362942,"id":"9130","kind":"VM","logical_id":"vm9130"},{"archive_size_bytes":3189,"id":"filebrowser","kind":"APP","logical_id":"app-filebrowser"},{"archive_size_bytes":156634,"id":"skladchik-reports-monitor","kind":"APP","logical_id":"app-skladchik-reports-monitor"},{"archive_size_bytes":270193548,"id":"immich","kind":"APP","logical_id":"app-immich"},{"archive_size_bytes":75598903,"id":"hbarhub","kind":"APP","logical_id":"app-hbarhub"},{"archive_size_bytes":8978589,"id":"control-bootstrap-v2","kind":"CONTROL","logical_id":"control-bootstrap-v2"}]}
|
|
{"clouds":[{"free_bytes":756548494504,"remote":"pve01-mail-01-crypt","total_bytes":1099511627776,"used_bytes":342963133272},{"free_bytes":763036822194,"remote":"pve01-mail-02-crypt","total_bytes":1099511627776,"used_bytes":336474805582}],"observed_at_epoch":1788644170,"schema":"homelab-backup-observability-snapshot-v1","targets":[{"archive_size_bytes":478357303,"id":"110","kind":"VM","logical_id":"vm110"},{"archive_size_bytes":369892675,"id":"111","kind":"VM","logical_id":"vm111"},{"archive_size_bytes":223672749,"id":"112","kind":"VM","logical_id":"vm112"},{"archive_size_bytes":221529138,"id":"113","kind":"VM","logical_id":"vm113"},{"archive_size_bytes":10591109796,"id":"130","kind":"VM","logical_id":"vm130"},{"archive_size_bytes":887338816,"id":"150","kind":"VM","logical_id":"vm150"},{"archive_size_bytes":3619770210,"id":"160","kind":"VM","logical_id":"vm160"},{"archive_size_bytes":2023943350,"id":"171","kind":"VM","logical_id":"vm171"},{"archive_size_bytes":1557482946,"id":"190","kind":"VM","logical_id":"vm190"},{"archive_size_bytes":10998362942,"id":"9130","kind":"VM","logical_id":"vm9130"},{"archive_size_bytes":3189,"id":"filebrowser","kind":"APP","logical_id":"app-filebrowser"},{"archive_size_bytes":156634,"id":"skladchik-reports-monitor","kind":"APP","logical_id":"app-skladchik-reports-monitor"},{"archive_size_bytes":270193548,"id":"immich","kind":"APP","logical_id":"app-immich"},{"archive_size_bytes":75598903,"id":"hbarhub","kind":"APP","logical_id":"app-hbarhub"},{"archive_size_bytes":8978589,"id":"control-bootstrap-v2","kind":"CONTROL","logical_id":"control-bootstrap-v2"}]}
|
|
{"clouds":[{"free_bytes":743574336966,"remote":"pve01-mail-01-crypt","total_bytes":1099511627776,"used_bytes":355937290810},{"free_bytes":743574273107,"remote":"pve01-mail-02-crypt","total_bytes":1099511627776,"used_bytes":355937354669}],"observed_at_epoch":1788649558,"schema":"homelab-backup-observability-snapshot-v1","targets":[{"archive_size_bytes":478357303,"id":"110","kind":"VM","logical_id":"vm110"},{"archive_size_bytes":369892675,"id":"111","kind":"VM","logical_id":"vm111"},{"archive_size_bytes":223672749,"id":"112","kind":"VM","logical_id":"vm112"},{"archive_size_bytes":221529138,"id":"113","kind":"VM","logical_id":"vm113"},{"archive_size_bytes":10588466039,"id":"130","kind":"VM","logical_id":"vm130"},{"archive_size_bytes":887338816,"id":"150","kind":"VM","logical_id":"vm150"},{"archive_size_bytes":3619770210,"id":"160","kind":"VM","logical_id":"vm160"},{"archive_size_bytes":6486807437,"id":"170","kind":"VM","logical_id":"vm170"},{"archive_size_bytes":2027528282,"id":"171","kind":"VM","logical_id":"vm171"},{"archive_size_bytes":1557482946,"id":"190","kind":"VM","logical_id":"vm190"},{"archive_size_bytes":10998362942,"id":"9130","kind":"VM","logical_id":"vm9130"},{"archive_size_bytes":3191,"id":"filebrowser","kind":"APP","logical_id":"app-filebrowser"},{"archive_size_bytes":157473,"id":"skladchik-reports-monitor","kind":"APP","logical_id":"app-skladchik-reports-monitor"},{"archive_size_bytes":270193543,"id":"immich","kind":"APP","logical_id":"app-immich"},{"archive_size_bytes":75669106,"id":"hbarhub","kind":"APP","logical_id":"app-hbarhub"},{"archive_size_bytes":8958389,"id":"control-bootstrap-v2","kind":"CONTROL","logical_id":"control-bootstrap-v2"}]}
|
|
{"clouds":[{"free_bytes":743574336966,"remote":"pve01-mail-01-crypt","total_bytes":1099511627776,"used_bytes":355937290810},{"free_bytes":743574273107,"remote":"pve01-mail-02-crypt","total_bytes":1099511627776,"used_bytes":355937354669}],"observed_at_epoch":1788653194,"schema":"homelab-backup-observability-snapshot-v1","targets":[{"archive_size_bytes":478357303,"id":"110","kind":"VM","logical_id":"vm110"},{"archive_size_bytes":369892675,"id":"111","kind":"VM","logical_id":"vm111"},{"archive_size_bytes":223672749,"id":"112","kind":"VM","logical_id":"vm112"},{"archive_size_bytes":221529138,"id":"113","kind":"VM","logical_id":"vm113"},{"archive_size_bytes":10588466039,"id":"130","kind":"VM","logical_id":"vm130"},{"archive_size_bytes":887338816,"id":"150","kind":"VM","logical_id":"vm150"},{"archive_size_bytes":3619770210,"id":"160","kind":"VM","logical_id":"vm160"},{"archive_size_bytes":6486807437,"id":"170","kind":"VM","logical_id":"vm170"},{"archive_size_bytes":2027528282,"id":"171","kind":"VM","logical_id":"vm171"},{"archive_size_bytes":1557482946,"id":"190","kind":"VM","logical_id":"vm190"},{"archive_size_bytes":10998362942,"id":"9130","kind":"VM","logical_id":"vm9130"},{"archive_size_bytes":3191,"id":"filebrowser","kind":"APP","logical_id":"app-filebrowser"},{"archive_size_bytes":157473,"id":"skladchik-reports-monitor","kind":"APP","logical_id":"app-skladchik-reports-monitor"},{"archive_size_bytes":270193543,"id":"immich","kind":"APP","logical_id":"app-immich"},{"archive_size_bytes":75669106,"id":"hbarhub","kind":"APP","logical_id":"app-hbarhub"},{"archive_size_bytes":8958389,"id":"control-bootstrap-v2","kind":"CONTROL","logical_id":"control-bootstrap-v2"}]}
|
|
{"clouds":[{"free_bytes":739949922534,"remote":"pve01-mail-01-crypt","total_bytes":1099511627776,"used_bytes":359561705242},{"free_bytes":739949858675,"remote":"pve01-mail-02-crypt","total_bytes":1099511627776,"used_bytes":359561769101}],"observed_at_epoch":1788656802,"schema":"homelab-backup-observability-snapshot-v1","targets":[{"archive_size_bytes":478357303,"id":"110","kind":"VM","logical_id":"vm110"},{"archive_size_bytes":369892675,"id":"111","kind":"VM","logical_id":"vm111"},{"archive_size_bytes":223672749,"id":"112","kind":"VM","logical_id":"vm112"},{"archive_size_bytes":221529138,"id":"113","kind":"VM","logical_id":"vm113"},{"archive_size_bytes":10588466039,"id":"130","kind":"VM","logical_id":"vm130"},{"archive_size_bytes":887338816,"id":"150","kind":"VM","logical_id":"vm150"},{"archive_size_bytes":3623527665,"id":"160","kind":"VM","logical_id":"vm160"},{"archive_size_bytes":6486807437,"id":"170","kind":"VM","logical_id":"vm170"},{"archive_size_bytes":2027528282,"id":"171","kind":"VM","logical_id":"vm171"},{"archive_size_bytes":1557482946,"id":"190","kind":"VM","logical_id":"vm190"},{"archive_size_bytes":10998362942,"id":"9130","kind":"VM","logical_id":"vm9130"},{"archive_size_bytes":3191,"id":"filebrowser","kind":"APP","logical_id":"app-filebrowser"},{"archive_size_bytes":157473,"id":"skladchik-reports-monitor","kind":"APP","logical_id":"app-skladchik-reports-monitor"},{"archive_size_bytes":270193543,"id":"immich","kind":"APP","logical_id":"app-immich"},{"archive_size_bytes":75669106,"id":"hbarhub","kind":"APP","logical_id":"app-hbarhub"},{"archive_size_bytes":8958389,"id":"control-bootstrap-v2","kind":"CONTROL","logical_id":"control-bootstrap-v2"}]}
|
|
HEALTH_FILE=homebox-backup.txt|mtime=2026-07-09 19:01:20.790193328 +0300|sha256=b388a7150af8169744f923e78e519408710c482afa7c2a3fcc334d1a1d3006e6
|
|
STATUS=OK TS=2026-07-09T16:01:19Z TYPE=homebox-backup JOB=homelab-pve01-app-homebox-20260709T051713Z SOURCE=latest-appbackup SIZE=8873 LOCAL_BACKUP_DELETED=true JSON=/var/lib/homelab-backup/apps/latest-homebox.json NOTE=service-readiness-backup-proof
|
|
HEALTH_FILE=edge-vm-vzdump-backup.txt|mtime=2026-08-30 21:08:08.621601440 +0300|sha256=35e1b1b24eea8e1ca6506ef437b49606f1e691544f70965db8d30b9b075f9b12
|
|
STATUS=FAIL
|
|
TS=2026-08-30T18:08:08Z
|
|
TYPE=backup-matrix-generated
|
|
JOB_ID=homelab_edge_vm_vzdump_backup
|
|
JOB_NAME=homelab-edge-vm-offhost-freshness
|
|
MATRIX_MODE=authoritative
|
|
TIMER=homelab-edge-vm-offhost-freshness.timer
|
|
SERVICE=homelab-edge-vm-offhost-freshness.service
|
|
TIMER_ENABLED=not-found
|
|
TIMER_ACTIVE=inactive
|
|
SOURCE_KIND=health_file
|
|
EVIDENCE_LEVEL=artifact_or_health
|
|
SERVICE_RESULT=success
|
|
SERVICE_EXEC_MAIN_STATUS=0
|
|
SOURCE_HEALTH=/var/lib/homelab-health/edge-vm-offhost-freshness.txt
|
|
SOURCE_STATUS=FAIL
|
|
SOURCE_MTIME_UTC=2026-08-08T01:50:13Z
|
|
AGE_SECONDS=1959475
|
|
MAX_AGE_SECONDS=172800
|
|
FRESHNESS=STALE
|
|
LEGACY_SOURCE_PATH=/var/lib/homelab-health/edge-vm-vzdump-backup.txt
|
|
CANONICAL_JOB_ID=homelab_edge_vm_vzdump_backup
|
|
CANONICAL_STATE=/var/lib/homelab-backup-matrix/state.json
|
|
HEALTH_FILE=npmplus-kuma-config-backup.txt|mtime=2026-08-30 21:08:08.440410577 +0300|sha256=be5d4aaace4430857d61615081e8a8ac9fcc577aed6224356cf154114ae08250
|
|
STATUS=FAIL
|
|
TS=2026-08-30T18:08:08Z
|
|
TYPE=backup-matrix-generated
|
|
JOB_ID=homelab_appbackup_npmplus_edge
|
|
JOB_NAME=homelab-appbackup-npmplus-edge
|
|
MATRIX_MODE=authoritative
|
|
TIMER=homelab-appbackup-npmplus-edge.timer
|
|
SERVICE=homelab-appbackup-npmplus-edge.service
|
|
TIMER_ENABLED=not-found
|
|
TIMER_ACTIVE=inactive
|
|
SOURCE_KIND=health_file
|
|
EVIDENCE_LEVEL=artifact_or_health
|
|
SERVICE_RESULT=success
|
|
SERVICE_EXEC_MAIN_STATUS=0
|
|
SOURCE_HEALTH=/var/lib/homelab-backup-matrix/evidence/homelab_appbackup_npmplus_edge/0-latest-npmplus-edge.json
|
|
SOURCE_STATUS=OK
|
|
SOURCE_MTIME_UTC=2026-08-06T21:21:34Z
|
|
AGE_SECONDS=2061994
|
|
MAX_AGE_SECONDS=172800
|
|
FRESHNESS=STALE
|
|
LEGACY_SOURCE_PATH=/var/lib/homelab-health/npmplus-kuma-config-backup.txt
|
|
CANONICAL_JOB_ID=homelab_appbackup_npmplus_edge
|
|
CANONICAL_STATE=/var/lib/homelab-backup-matrix/state.json
|
|
HEALTH_FILE=backup-coverage-matrix.txt|mtime=2026-08-30 21:08:08.539509156 +0300|sha256=48b3b20a0dc225b93e0f7b8b1c2746649eb6f965d12f996030cc8ec8791604ff
|
|
STATUS=FAIL
|
|
TS=2026-08-30T18:08:08Z
|
|
TYPE=backup-matrix-generated
|
|
JOB_ID=homelab_backup_coverage_matrix
|
|
JOB_NAME=homelab-backup-coverage-matrix
|
|
MATRIX_MODE=authoritative
|
|
TIMER=homelab-backup-coverage-matrix.timer
|
|
SERVICE=homelab-backup-coverage-matrix.service
|
|
TIMER_ENABLED=not-found
|
|
TIMER_ACTIVE=inactive
|
|
SOURCE_KIND=health_file
|
|
EVIDENCE_LEVEL=artifact_or_health
|
|
SERVICE_RESULT=success
|
|
SERVICE_EXEC_MAIN_STATUS=0
|
|
SOURCE_HEALTH=/var/lib/homelab-backup-matrix/evidence/homelab_backup_coverage_matrix/0-backup-coverage-matrix.txt
|
|
SOURCE_STATUS=OK
|
|
SOURCE_MTIME_UTC=2026-08-06T21:34:03Z
|
|
AGE_SECONDS=2061245
|
|
MAX_AGE_SECONDS=172800
|
|
FRESHNESS=STALE
|
|
LEGACY_SOURCE_PATH=/var/lib/homelab-health/backup-coverage-matrix.txt
|
|
CANONICAL_JOB_ID=homelab_backup_coverage_matrix
|
|
CANONICAL_STATE=/var/lib/homelab-backup-matrix/state.json
|
|
HEALTH_FILE=vm160-remote-backup.txt|mtime=2026-07-21 02:39:14.312087060 +0300|sha256=de5897d501d5bc25ca6fe964bf2643b2f8cac6957f9f577a1ae0cf4c68c3587e
|
|
STATUS=FAIL
|
|
SERVICE=homelab-vm160-remote-backup
|
|
TIMESTAMP_UTC=20260720T233913Z
|
|
VMID=160
|
|
TARGET_DIR=/mnt/staging/vzdump/vm160-pve02-20260720T233913Z
|
|
RC=1
|
|
HEALTH_FILE=vm-backup.txt|mtime=2026-07-24 07:43:04.241966168 +0300|sha256=78f0d0c132ea4b61be884501f342913fc13805900c083d82a59ddd30d067d980
|
|
STATUS=OK TS=2026-07-24T04:43:04Z TYPE=vm-backup BAD=0 auto-vmct=OK edge-vm-backup=OK edge-vm-offhost=OK edge-vm-restore=OK auto-timer=active edge-freshness-timer=active
|
|
HEALTH_FILE=app-stacks-batch-backup.txt|mtime=2026-07-09 10:22:35.530260317 +0300|sha256=9218b8b34426e4ce79310c8713e165e9e1ebee79f426f29eb2efe155076b3c36
|
|
STATUS=OK TS=2026-07-09T07:19:28Z TYPE=app-stacks-batch-backup SOURCE=backup-sla STATUS_SOURCE=OK NOTE=legacy-health-refresh-for-prometheus-readiness
|
|
HEALTH_FILE=blackbox-exporter-backup.txt|mtime=2026-07-09 11:04:19.961323675 +0300|sha256=2c46d63ab44ad8bd8f161a547103d8aef937643044b8b326e2b1ff609e9178d0
|
|
STATUS=OK TS=2026-07-09T08:04:14Z TYPE=blackbox-exporter-backup JOB=homelab-pve01-app-blackbox-exporter-20260709T080414Z SOURCE=blackbox_exporter_config SIZE=512 SHA256=31d9f2a61d820629c0e173c0e1a24cd65e24d1ef55ba0670d7fda06225cb8146 LOCAL_BACKUP_DELETED=true NOTE=service-readiness-backup-proof
|
|
HEALTH_FILE=node-exporter-backup.txt|mtime=2026-07-09 11:14:05.435232618 +0300|sha256=9312b7986eeb48f06f3f3722fcf970624a423b4c398652d83226e59aaff4fefa
|
|
STATUS=OK TS=2026-07-09T08:13:40Z TYPE=node-exporter-backup JOB=homelab-pve01-app-node-exporter-20260709T081340Z SOURCE=multi_host_node_exporter_config_textfile SIZE=19018 SHA256=bb7db78e6e79c104f89c7735922e2a2e311fba242d34728e64cd60cefe69e647 LOCAL_BACKUP_DELETED=true PVE01_TEXTFILE_REPAIR=OK NOTE=service-readiness-backup-proof
|
|
HEALTH_FILE=stirling-pdf-backup.txt|mtime=2026-07-09 21:08:55.532973745 +0300|sha256=d292f237702fbead21f1597ac86dd610f8d3a6f70b81d18d2a3e3bbd8aced551
|
|
STATUS=OK TS=2026-07-09T18:08:38Z TYPE=stirling-pdf-backup BAD=0 APP=stirling-pdf JOB=homelab-pve01-app-stirling-pdf-20260709T180838Z SIZE=40071 LOCAL_BACKUP_DELETED=true ENGINE=files JSON=/var/lib/homelab-backup/apps/latest-stirling-pdf.json
|
|
HEALTH_FILE=backup-audit.json|mtime=2026-09-23 23:47:36.448063169 +0300|sha256=b193b9fc04ed47f247d4c981468387ceff69cf1415e4295ec5b4de9e08d1b6fe
|
|
{"cloud":{"A":{"free":592249131673,"free_fraction":0.538647,"readable":true,"top_dirs":27,"total":1099511627776,"used":507262496103},"B":{"free":592249067814,"free_fraction":0.538647,"readable":true,"top_dirs":27,"total":1099511627776,"used":507262559962}},"cloud_b_restore_drill":true,"dr_proven":true,"generated_at_epoch":1790196452,"issues":[{"code":"RPO_BREACH","logical_id":"vm150","root_cause":"NO_FRESH_V2_OR_SEALED_QUORUM","severity":"HIGH"},{"code":"SOURCE_OFFHOST_MISSING","logical_id":"-","root_cause":"CR_BRANCH_SHA_NOT_EXACT","severity":"MEDIUM"}],"schema":"homelab-backup-audit-v3","source":{"local_head":"6401ea1e99988b9d2550de544c9e45bd7e829bdd","pushurl_disabled":true,"remote_head":null},"status":"RED","targets":[{"decision":"SKIP_FRESH","generation":"app-filebrowser-pve01-20260923T143005Z","kind":"APP","legacy_age_seconds":22640,"logical_id":"app-filebrowser","state":"FRESH"},{"decision":"SKIP_FRESH","generation":"app-hbarhub-pve01-20260923T143158Z","kind":"APP","legacy_age_seconds":22468,"logical_id":"app-hbarhub","state":"FRESH"},{"decision":"SKIP_FRESH","generation":"app-immich-pve01-20260923T143041Z","kind":"APP","legacy_age_seconds":22568,"logical_id":"app-immich","state":"FRESH"},{"decision":"SKIP_FRESH","generation":"app-skladchik-reports-monitor-pve01-20260923T143025Z","kind":"APP","legacy_age_seconds":22618,"logical_id":"app-skladchik-reports-monitor","state":"FRESH"},{"age_seconds":44470,"generation":"control-bootstrap-v2-pve01-20260916T120057Z","kind":"CONTROL","logical_id":"control-bootstrap-v2","state":"FRESH"},{"generation":"v2-ct110-20260923T125546Z-34408ffe4182","kind":"VM","legacy_age_seconds":1696083,"logical_id":"vm110","replica_count":2,"source":"V2","state":"FRESH","state_id":"ct110","v2_age_seconds":28097},{"generation":"v2-ct111-20260923T133222Z-0d3f5fae9c35","kind":"VM","legacy_age_seconds":null,"logical_id":"vm111","replica_count":2,"source":"V2","state":"FRESH","state_id":"ct111","v2_age_seconds":25955},{"generation":"v2-ct112-20260923T200600Z-c1b47cc67dad","kind":"VM","legacy_age_seconds":1734578,"logical_id":"vm112","replica_count":2,"source":"V2","state":"FRESH","state_id":"ct112","v2_age_seconds":2422},{"generation":"v2-ct113-20260923T203800Z-bd913a8e8930","kind":"VM","legacy_age_seconds":null,"logical_id":"vm113","replica_count":2,"source":"V2","state":"FRESH","state_id":"ct113","v2_age_seconds":497},{"generation":"v2-vm130-20260923T011558Z-c9498d37d3d7","kind":"VM","legacy_age_seconds":null,"logical_id":"vm130","replica_count":2,"source":"V2","state":"FRESH","state_id":"vm130","v2_age_seconds":69146},{"generation":"vm150-pve01-20260904T053517Z","kind":"VM","legacy_age_seconds":1696199,"logical_id":"vm150","replica_count":2,"source":"NONE","state":"STALE","state_id":"vm150","v2_age_seconds":74828},{"generation":"v2-vm160-20260923T014608Z-f76d7671af5a","kind":"VM","legacy_age_seconds":null,"logical_id":"vm160","replica_count":2,"source":"V2","state":"FRESH","state_id":"vm160","v2_age_seconds":68046},{"generation":"v2-vm170-20260923T035958Z-923adacbc94b","kind":"VM","legacy_age_seconds":640820,"logical_id":"vm170","replica_count":2,"source":"V2","state":"FRESH","state_id":"vm170","v2_age_seconds":59695},{"generation":"v2-vm171-20260923T021946Z-4d6e4f578075","kind":"VM","legacy_age_seconds":670352,"logical_id":"vm171","replica_count":2,"source":"V2","state":"FRESH","state_id":"vm171","v2_age_seconds":66249},{"generation":"v2-vm190-20260923T032759Z-314d9ba2ec35","kind":"VM","legacy_age_seconds":null,"logical_id":"vm190","replica_count":2,"source":"V2","state":"FRESH","state_id":"vm190","v2_age_seconds":62166},{"generation":"v2-vm9130-20260923T071932Z-c0492db6535b","kind":"VM","legacy_age_seconds":null,"logical_id":"vm9130","replica_count":2,"source":"V2","state":"FRESH","state_id":"vm9130","v2_age_seconds":47261}]}
|
|
HEALTH_FILE=crypto-portfolio-ui-preapply-backup.txt|mtime=2026-07-13 10:42:57.148769166 +0300|sha256=7b4b3faf518b57a01b5bb1302377fc1fb13ad9a206f9e8211edfd8bbbc0bf395
|
|
STATUS=OK TS=2026-07-13T07:42:53Z TYPE=crypto-portfolio-ui-preapply-backup BAD=0 COMMAND_PHYSICAL_LINES=1 PREVIOUS_FAILURE=CORRECTED PREVIOUS_FAILURE_ROOT_CAUSE=INLINE_PYTHON_QUOTE_ERROR_IN_NONESSENTIAL_HELPER_PROOF BACKUP_SERVICE_RERUN=NO MANUAL_REHYDRATE_RERUN=NO CANONICAL_RESTORE_CHECK_RERUN=YES_READ_ONLY PRODUCTION_CHANGED=NO PRODUCTION_DATABASE_CHANGED=NO DESIRED_STATE_CHANGED=NO SERVICE_RESTARTED=NO CONTAINER_RECREATED=NO CLOUD_WRITE_RUN=NO_READ_ONLY_RESTORE_CHECK TARGET_HOST=core-apps.gram1.ru INPUT_INDEX_SHA256=8bbe88f5f420fe429d479c598e1f3f904521eb166d4707e32e579851e84c205e CANDIDATE_INDEX_SHA256=e497a36d8549654c6d451360a11a2ad714e19c23e6d384330a5436f7d32267cb BACKUP_JOB=homelab-pve01-app-crypto-portfolio-20260713T071127Z BACKUP_STATUS=success BACKUP_SCHEMA=homelab-app-backup-manifest-v2-flat BACKUP_LAYOUT=flat-v1 BACKUP_PATH_REL=hb/a/cp-260713-071127 BACKUP_DATABASE_INTEGRITY=ok BACKUP_FILE_COUNT=19 BACKUP_ARCHIVE_SIZE_BYTES=136632 BACKUP_ARCHIVE_SHA256=b29eb55820ec46c474f51e995e92e371251b6df717ac98182d807d1dbf8473f5 BACKUP_LOCAL_STAGING_DELETED=YES CANONICAL_TWO_CLOUD_RESTORE_CHECK=OK APPBACKUPCTL_RESTORE_CHECK=OK_BOTH_CLOUDS ROLLBACK_PATH=/var/lib/homelab-incidents/crypto-ui-center-alert-currency-prechange-20260713T071125Z ROLLBACK_MANIFEST_SHA256=0ef56c8f021f549777131ffe878cd5a263bdfb6f1dfb8bb080cefdd554573aca ROLLBACK_INDEX_SHA256=8bbe88f5f420fe429d479c598e1f3f904521eb166d4707e32e579851e84c205e PRODUCTION_UNCHANGED=YES PRODUCTION_HEALTH=200 VM170_FAILED_UNITS=0 PVE01_UNRELATED_FAILED_BASELINE_UNCHANGED=YES CHART_HEALTH_TIMER=ACTIVE BACKUP_TIMER=ACTIVE DESIRED_STATE_HEAD=80763a6c62b94cde4f3293f3b529980f76919d25 DESIRED_STATE_CLEAN=YES DESIRED_STATE_REMOTE_COUNT=0 READY_FOR_UI_PRODUCTION_APPLY=YES PREAPPLY_ROOT=/var/lib/homelab-change-sets/crypto-portfolio-ui-preapply-backup-canonical-close/20260713T074253Z
|
|
HEALTH_FILE=backup-matrix-vm-health-feedback-fix.txt|mtime=2026-07-13 20:29:02.896780731 +0300|sha256=b359a93e09405221e25670600e4b45503ac30c7a8ada2f4e06f0cd9734a6e803
|
|
STATUS=OK TS=2026-07-13T17:29:02Z TYPE=backup-matrix-vm-health-feedback-fix BAD=0 VM_BACKUP_HEALTH_TIMER=homelab-edge-vm-offhost-freshness.timer OLD_TIMER_REENABLED=NO MATRIX_STATUS=OK JOBS_TOTAL=35 JOBS_BAD=0 JOBS_UNMAPPED=0 JOBS_DISABLED=4 READ_WRITE_OVERLAP=0 HEAVY_BACKUP_RUN=NO LIVE_SCRIPT_SHA256=a795ad5cacd48ac72e85c3a7908ad456d1e7c563ef4399e6957725146463faf2 MATRIX_CONFIG_SHA256=7613bcb25d32e61fac114da13af81e38354ec22a012a6deeaa168ee4cc454f87 MATRIX_STATE_SHA256=75f8c749e25df7842ea3be2679119b922c94c2f5ea99700b56c823cfbaedd539 ROLLBACK=/var/lib/homelab-incidents/vm-backup-health-matrix-feedback-fix-20260713T172901Z
|
|
HEALTH_FILE=beszel-two-cloud-backup.txt|mtime=2026-07-10 22:05:25.857040628 +0300|sha256=31278a684937710c66cdabb2c4eff0c0fda078e7fef976ab8f4c49ea5778a5ca
|
|
STATUS=OK TS=2026-07-10T19:05:25Z TYPE=beszel-two-cloud-backup BAD=0 JOB=homelab-pve01-app-beszel-20260710T190453Z ARCHIVE_SIZE_BYTES=20068 LOCAL_STAGING_DELETED=true OFFHOST=OK RESTORE=OK
|
|
HEALTH_FILE=cluster-admin-incident-engine-backup.txt|mtime=2026-07-11 02:30:49.625263708 +0300|sha256=5f56ac6b48e94a81f7a9ffa073525dc255b1fede76fcc04f7b629e0f51cb98ed
|
|
STATUS=OK TS=2026-07-10T23:30:49Z TYPE=cluster-admin-incident-engine-backup BAD=0 JOB=homelab-pve01-app-cluster-admin-incident-engine-20260710T232958Z ARCHIVE_SIZE_BYTES=157380 LOCAL_STAGING_DELETED=true OFFHOST=OK RESTORE=OK POST_FIX_CODE=included
|
|
HEALTH_FILE=backup-unified-audit.json|mtime=2026-09-21 07:36:42.412067103 +0300|sha256=5da55c119b77511b150a6c2c7380842551293e5d3d12e807ab25291a4d9031b6
|
|
{
|
|
"schema": "homelab-backup-unified-audit-v1",
|
|
"task_id": "backup-unified-monitoring-v1",
|
|
"change_id": "CR-2026-0032",
|
|
"updated_at_utc": "2026-09-21T04:36:42Z",
|
|
"audit_status": "OK",
|
|
"backup_status": "RED",
|
|
"risk_severity": "CRITICAL",
|
|
"risk_reasons": [
|
|
"SOURCE_DRIFT",
|
|
"INFRASTRUCTURE_INVARIANT",
|
|
"RPO_BREACH",
|
|
"CAPACITY_CRITICAL",
|
|
"RESUME_PENDING"
|
|
],
|
|
"dr_readiness": "PROVEN",
|
|
"dr_proof_status": "PROVEN",
|
|
"control_freshness": "STALE_PROVEN",
|
|
"lifecycle_status": "CLOSED_PROVEN",
|
|
"vm_count": 11,
|
|
"app_count": 4,
|
|
"fresh_count": 4,
|
|
"stale_count": 10,
|
|
"resume_count": 1,
|
|
"unverified_count": 0,
|
|
"cloud_ok_count": 2,
|
|
"scheduler_ok_count": 6,
|
|
"source_ok_count": 28,
|
|
"source_drift_count": 1,
|
|
"payload_leaks": 0,
|
|
"control_status": "STALE_PROVEN",
|
|
"control_source_coverage": "RESTORE_EVIDENCE",
|
|
"staging_status": "OK",
|
|
"staging_bad_count": 0,
|
|
"staging_warn_count": 0,
|
|
"staging_node_count": 3,
|
|
"escrow_status": "OK",
|
|
"retention_status": "ACTIVE_OWNER_AWARE",
|
|
"historical_gc_proven": false,
|
|
"owner_aware_gc_proven": true,
|
|
"observability_status": "ATTENTION",
|
|
"size_anomaly_count": 0,
|
|
"size_learning_count": 0,
|
|
"capacity_status": "CRIT",
|
|
"capacity_worst_used_percent": 74,
|
|
"capacity_worst_days_to_80": 3,
|
|
"observability_history_snapshot_count": 336,
|
|
"provider_worm_proven": false,
|
|
"provider_worm_skipped_by_policy": true,
|
|
"clean_machine_bootstrap_proven": true,
|
|
HEALTH_FILE=legacy-monitoring-backups-migrated.txt|mtime=2026-07-10 23:00:03.116834953 +0300|sha256=3d8a60e0d8aeb25aed50f6eb2a92df4ef2f544b7409711f0afec378099b635c7
|
|
STATUS=OK TS=2026-07-10T20:00:03Z TYPE=legacy-monitoring-backups-migrated BAD=0 APPS=6 RESTORE_FAILURES=0 ACTIVE_LEGACY_STATES=0 OFFHOST=OK RESTORE=OK
|
|
HEALTH_FILE=backup-boot-drill.json|mtime=2026-09-20 07:40:22.416980264 +0300|sha256=5cf6ed79fdec1f2901a868b05a57f17d1388189dcc2fb32e191b0ee4fd1f5d97
|
|
{
|
|
"drill_executed": false,
|
|
"reason": "BACKUP_LOCK_BUSY",
|
|
"schema": "homelab-backup-boot-drill-health-v1",
|
|
"status": "DEFERRED",
|
|
"updated_at_utc": "2026-09-20T04:40:22Z"
|
|
}
|
|
HEALTH_FILE=vm170-vm171-full-backup.txt|mtime=2026-08-30 21:08:08.908433073 +0300|sha256=dec2afb6c8b47c0c6be4f5a98e226ac3285524bc997cd93dd73942314e4367ee
|
|
STATUS=FAIL
|
|
TS=2026-08-30T18:08:08Z
|
|
TYPE=backup-matrix-generated
|
|
JOB_ID=homelab_vm_full_backup
|
|
JOB_NAME=homelab-vm-full-backup
|
|
MATRIX_MODE=authoritative
|
|
TIMER=homelab-vm-full-backup.timer
|
|
SERVICE=homelab-vm-full-backup.service
|
|
TIMER_ENABLED=not-found
|
|
TIMER_ACTIVE=inactive
|
|
SOURCE_KIND=health_file
|
|
EVIDENCE_LEVEL=artifact_or_health
|
|
SERVICE_RESULT=success
|
|
SERVICE_EXEC_MAIN_STATUS=0
|
|
SOURCE_HEALTH=/var/lib/homelab-backup-matrix/evidence/homelab_vm_full_backup/0-vm170-vm171-full-backup.txt
|
|
SOURCE_STATUS=OK
|
|
SOURCE_MTIME_UTC=2026-07-24T00:10:18Z
|
|
AGE_SECONDS=3261470
|
|
MAX_AGE_SECONDS=172800
|
|
FRESHNESS=STALE
|
|
LEGACY_SOURCE_PATH=/var/lib/homelab-health/vm170-vm171-full-backup.txt
|
|
CANONICAL_JOB_ID=homelab_vm_full_backup
|
|
CANONICAL_STATE=/var/lib/homelab-backup-matrix/state.json
|
|
HEALTH_FILE=uptime-kuma-backup.txt|mtime=2026-07-09 23:39:50.594635691 +0300|sha256=62270c278bd63d2068eca9c0ee8a8036436328334a9393b357353eb8d52cc079
|
|
STATUS=OK TS=2026-07-09T20:39:29Z TYPE=uptime-kuma-backup BAD=0 APP=uptime-kuma JOB=homelab-pve01-app-uptime-kuma-20260709T072138Z SIZE=99851064 LOCAL_BACKUP_DELETED=true JSON=/var/lib/homelab-backup/apps/latest-uptime-kuma.json
|
|
HEALTH_FILE=calibre-web-backup.txt|mtime=2026-07-09 21:46:20.933829215 +0300|sha256=2e841d7edd46668242924dae527480ead9c298abeb30a4c96734c382b09c90d7
|
|
STATUS=OK TS=2026-07-09T18:46:20Z TYPE=calibre-web-backup BAD=0 APP=calibre-web JOB=homelab-pve01-app-calibre-web-20260709T051713Z SIZE=6630 LOCAL_BACKUP_DELETED=true JSON=/var/lib/homelab-backup/apps/latest-calibre-web.json
|
|
HEALTH_FILE=syncthing-backup.txt|mtime=2026-07-09 22:55:13.502354244 +0300|sha256=f9afb07a38c097b42a37b5a710917fafa6d31c943953d8daf6db72b7b91d9c24
|
|
STATUS=OK TS=2026-07-09T19:55:12Z TYPE=syncthing-backup BAD=0 APP=syncthing JOB=homelab-pve01-app-syncthing-20260709T051713Z SIZE=6732 LOCAL_BACKUP_DELETED=true JSON=/var/lib/homelab-backup/apps/latest-syncthing.json
|
|
HEALTH_FILE=dozzle-backup.txt|mtime=2026-07-09 11:32:11.068758347 +0300|sha256=cc3f0d96c68c610cad9c1c38c70c89387617d43e1df6e0d304e4b7ae6b0464bb
|
|
STATUS=OK TS=2026-07-09T08:32:09Z TYPE=dozzle-backup JOB=homelab-pve01-app-dozzle-20260709T083209Z SOURCE=dozzle_stack_config SIZE=1328 SHA256=a571ee6f6012b25c16cdf15693dba29d8dd3acc3ce872b52f5c34edb81aa1ec8 LOCAL_BACKUP_DELETED=true NOTE=service-readiness-backup-proof
|
|
HEALTH_FILE=vm-backup-policy.txt|mtime=2026-08-30 21:08:08.889182826 +0300|sha256=947eef16fbfd3815d6f823c3bc990415a33c5611342bc0d32fe959cb9e95b799
|
|
STATUS=FAIL
|
|
TS=2026-08-30T18:08:08Z
|
|
TYPE=backup-matrix-generated
|
|
JOB_ID=homelab_vm_backup_policy
|
|
JOB_NAME=homelab-vm-backup-policy
|
|
MATRIX_MODE=authoritative
|
|
TIMER=homelab-vm-backup-policy.timer
|
|
SERVICE=homelab-vm-backup-policy.service
|
|
TIMER_ENABLED=not-found
|
|
TIMER_ACTIVE=inactive
|
|
SOURCE_KIND=health_file
|
|
EVIDENCE_LEVEL=artifact_or_health
|
|
SERVICE_RESULT=success
|
|
SERVICE_EXEC_MAIN_STATUS=0
|
|
SOURCE_HEALTH=/var/lib/homelab-backup-matrix/evidence/homelab_vm_backup_policy/0-vm-backup-policy.txt
|
|
SOURCE_STATUS=OK
|
|
SOURCE_MTIME_UTC=2026-07-24T05:26:24Z
|
|
AGE_SECONDS=3242504
|
|
MAX_AGE_SECONDS=172800
|
|
FRESHNESS=STALE
|
|
LEGACY_SOURCE_PATH=/var/lib/homelab-health/vm-backup-policy.txt
|
|
CANONICAL_JOB_ID=homelab_vm_backup_policy
|
|
CANONICAL_STATE=/var/lib/homelab-backup-matrix/state.json
|
|
HEALTH_FILE=netbird-peers.txt|mtime=2026-09-23 23:58:22.533880166 +0300|sha256=77fdd7924a7623499cc18560449d2190a7123fe7f0277c469a8a1360f440c869
|
|
STATUS=ERROR
|
|
SERVICE=netbird-peers
|
|
TIMESTAMP_UTC=20260923T205759Z
|
|
ATTEMPT_USED=2
|
|
pve01 service=active/enabled wt0_ip=[IP]/16 management=no signal=no peers=2/11 Connected
|
|
pve02 service=active/enabled wt0_ip=[IP]/16 management=no signal=no peers=2/11 Connected
|
|
pve03 service=active/enabled wt0_ip=[IP]/16 management=no signal=no peers=2/11 Connected
|
|
edge-vm service=active/enabled wt0_ip=[IP]/16 management=yes signal=yes peers=2/11 Connected
|
|
HEALTH_FILE=hapusya-site-backup-path-mapping.txt|mtime=2026-07-13 15:10:58.138221543 +0300|sha256=fb9db1095347b4db4dc7e37d1085cb758a5608ff8c6f3b934da00f44a905142c
|
|
STATUS=OK TS=2026-07-13T12:10:58Z TYPE=hapusya-site-backup-path-mapping BAD=0 MAP_SHA256=815b52dcedf7e3e92e07939bade32c7e8ed6acc0f9e064b65db6daec9f559c63 SHORT=hps LEGACY_STATE_PATH=pve01/apps/hapusya-site/homelab-pve01-app-hapusya-site-20260713T115259Z DESIRED_STATE_SNAPSHOT=/srv/homelab-desired-state/config/homelab-backup-paths.map RESTORE_CHECK=OK_BOTH_CLOUDS RETENTION=OK PRODUCTION_CHANGED=YES_COMMIT_POINT_PRESERVED
|
|
HEALTH_FILE=crowdsec-capi-netbird-final.txt|mtime=2026-07-02 22:30:48.260143650 +0300|sha256=20b03407e972e39c3a8b7e8ca008bf4a4b6928ed3e0e9a822be886c3cd63fae0
|
|
STATUS=OK
|
|
TS=20260702T193048Z
|
|
CHECK=crowdsec-capi-netbird-final
|
|
DETAIL=CrowdSec CAPI/LAPI stable through NetBird exit route relay [IP]; WARP absent; route smoke OK; documentation and error register updated.
|
|
EDGE=edge-vm
|
|
EDGE_IP=[IP]
|
|
EDGE_NETBIRD_IP=[IP]
|
|
PRIMARY_EXIT=relay.netbird.selfhosted
|
|
PRIMARY_EXIT_NETBIRD_IP=[IP]
|
|
BACKUP_EXIT=mail.netbird.selfhosted
|
|
BACKUP_EXIT_NETBIRD_IP=[IP]
|
|
TRACE_IP=[IP]
|
|
TRACE_COLO=OTP
|
|
PROOFS=771AC_REGISTER_CROWDSEC_CAPI_AFTER_EXIT_ROUTE,771AD_FINAL_CROWDSEC_CAPI_NETBIRD_CLOSURE,771AE_CROWDSEC_CAPI_SOAK_FINAL,771AF_CROWDSEC_CAPI_PROOF_INDEX
|
|
HEALTH_FILE=cluster-admin-stage3-preapply-backup.txt|mtime=2026-07-13 06:15:56.239623410 +0300|sha256=95632c844cbdafe9be502b9164b1ec4351e55a1a0f1ae55668426c8affaa657c
|
|
STATUS=OK TS=2026-07-13T03:15:54Z TYPE=cluster-admin-stage3-preapply-backup BAD=0 COMMAND_PHYSICAL_LINES=1 PRODUCTION_CHANGED=EVIDENCE_ONLY LIVE_APPLICATION_CODE_CHANGED=NO DESIRED_STATE_CHANGED=NO SERVICE_RESTARTED=NO CLOUD_WRITE_RUN=NO_REUSE_VERIFIED_EXISTING_BACKUP APP_PRECHANGE_SHA256=c2748b8f96df842f491b5d2790790594222549853d0e2336cf60ecd6d4de0020 APP_CANDIDATE_SHA256=672d6a2975dd0e093788e7eb81a76e7f901ea6374ca4092a25ba0091aba1cb50 ROLLBACK_CHECKPOINT=VERIFIED REMOTE_CHECKPOINT_DIR=/var/lib/cluster-admin-stage3-preapply/20260713T030540Z CHECKPOINT_MANIFEST_SHA256=ede8c349a3456384f11460477d7cabc1a2aa0d18189ef3aa772471f2ebb26c77 CHECKPOINT_VALIDATION_WORKDIR=CORRECTED APPBACKUP=VERIFIED APPBACKUP_JOB=homelab-pve01-app-cluster-admin-incident-engine-20260713T031116Z APPBACKUP_SIZE_BYTES=1767199 APPBACKUP_RUN_LOG_EMPTY=ACCEPTED APPBACKUP_RESTORE_CHECK=OK APPBACKUP_ROUTING=VERIFIED LIVE_APP_UNCHANGED=YES PRODUCTION_RUNTIME_HEALTH=OK DESIRED_STATE_HEAD=8f888e5f2063e49da918b31584c9ea4bcf53af3a PREAPPLY_ROOT=/var/lib/homelab-stage3-preapply-appbackup/20260713T031116Z READY_FOR_STAGE3_PRODUCTION_APPLY=YES
|
|
HEALTH_FILE=node-red-backup.txt|mtime=2026-07-09 22:46:45.732436363 +0300|sha256=cf53c240e74a9f8670ab0af52681ce9de4e45f332e70fc142925c357eaf9f00c
|
|
STATUS=OK TS=2026-07-09T19:46:42Z TYPE=node-red-backup BAD=0 APP=node-red JOB=homelab-pve01-app-node-red-20260709T051713Z SIZE=10521 LOCAL_BACKUP_DELETED=true JSON=/var/lib/homelab-backup/apps/latest-node-red.json
|
|
HEALTH_FILE=proxmox-pve01-backup.txt|mtime=2026-07-09 15:29:41.985138298 +0300|sha256=2b8ca75eeec121d7e2b0c6a309d43798da5f864b57fef327f37a56b9ced3b5a8
|
|
STATUS=OK TS=2026-07-09T12:29:38Z TYPE=proxmox-pve01-backup JOB=homelab-pve01-app-proxmox-pve01-20260709T122938Z SOURCE=safe_pve01_config_facts SIZE=4314 LOCAL_BACKUP_DELETED=true JSON=/var/lib/homelab-backup/apps/latest-proxmox-pve01.json NOTE=service-readiness-backup-proof
|
|
HEALTH_FILE=service-readiness-backup-health.txt|mtime=2026-07-09 09:41:40.097970342 +0300|sha256=f8074ea30fd108c0db7c85caebaaaa4a3600ea49fbea717f372bf5e6218753b4
|
|
STATUS=OK TS=2026-07-09T06:41:40Z TYPE=service-readiness-final BAD=0 SERVICE=backup-health HOMEPAGE=OK NPMPLUS=OK KUMA=OK ENDPOINTS=LOCAL_LAN_PUBLIC_OK CORE_GATES=OK LISTENERS=OK URL=https://backup.gram1.ru/backup.txt OVERALL=https://backup.gram1.ru/overall.txt
|
|
HEALTH_FILE=hapusya-site-backup.txt|mtime=2026-07-13 15:10:58.137221528 +0300|sha256=5f8d4f49cf5cf6dba74912589dcd3c7fa0542171bfa31c649f6d94292fb8bb64
|
|
STATUS=OK TS=2026-07-13T12:10:58Z TYPE=hapusya-site-backup JOB=homelab-pve01-app-hapusya-site-20260713T115259Z ARCHIVE_SHA256=bf284292974600384c43c47fc99be6fedf0ebd68f722b402a7260f1df680d167 ARCHIVE_SIZE_BYTES=7895834 LOCAL_BACKUP_DELETED=true TWO_CLOUDS=OK ROUTE_SHA256=02d65cb0082a04cae282126cc829a002f171e863291c18ba04d788dc2d6282ca UNIT_SHA256=b3b07137b25ac0a0b4bdadadad654cf209387ade86a2ef2cdca58e8a7c0cc212 RELATIVE_TREE_SHA256=9dadf21da041bc4d6ee4a42f5b3ac7719c16bcc39052de32af895c6cd28286cb
|
|
HEALTH_FILE=backup.txt|mtime=2026-07-24 07:04:41.721444174 +0300|sha256=88d791f9955261ebe5a65303dc418c70dca25ba23c29a92769a6785a9cd9e5bb
|
|
STATUS=OK
|
|
REASON=latest backup exists, fresh, complete, and sha256 verified
|
|
LATEST=/mnt/staging/vzdump/auto-full-backup-20260719T004537Z
|
|
AGE_HOURS=123
|
|
START=2026-07-24T07:04:28+03:00
|
|
END=2026-07-24T07:04:41+03:00
|
|
RC=0
|
|
HEALTH_FILE=npmplus-edge-backup.txt|mtime=2026-07-09 13:02:56.571425663 +0300|sha256=83e492c51913e884cad62305314e153c49bad522b9d65cddaefe337b7ed10e2d
|
|
STATUS=OK TS=2026-07-09T10:02:52Z TYPE=npmplus-edge-backup JOB=homelab-pve01-app-npmplus-edge-20260709T071928Z SOURCE=latest-appbackup SIZE=50316 LOCAL_BACKUP_DELETED=true JSON=/var/lib/homelab-backup/apps/latest-npmplus-edge.json NOTE=service-readiness-backup-proof
|
|
HEALTH_FILE=p1-safe-data-backup.txt|mtime=2026-07-07 01:09:37.332826625 +0300|sha256=c1f7e49cbb188b0c1b13aed8508d267d47212490e92b6cb85a3b70ab167fd13d
|
|
HEALTH_FILE=gitea-backup.txt|mtime=2026-08-30 21:08:08.480366809 +0300|sha256=5474fdacc557aef613bf14e7830fac12d9e616320519074a228a0a81ff6e8e68
|
|
STATUS=FAIL
|
|
TS=2026-08-30T18:08:08Z
|
|
TYPE=backup-matrix-generated
|
|
JOB_ID=homelab_appbackup_sqlite_batch
|
|
JOB_NAME=homelab-appbackup-sqlite-batch
|
|
MATRIX_MODE=authoritative
|
|
TIMER=homelab-appbackup-sqlite-batch.timer
|
|
SERVICE=homelab-appbackup-sqlite-batch.service
|
|
TIMER_ENABLED=not-found
|
|
TIMER_ACTIVE=inactive
|
|
SOURCE_KIND=systemd
|
|
EVIDENCE_LEVEL=execution_only
|
|
SERVICE_RESULT=success
|
|
SERVICE_EXEC_MAIN_STATUS=0
|
|
SOURCE_HEALTH=systemd:homelab-appbackup-sqlite-batch.service
|
|
SOURCE_STATUS=OK
|
|
SOURCE_MTIME_UTC=2026-07-24T01:46:28Z
|
|
AGE_SECONDS=3255700
|
|
MAX_AGE_SECONDS=172800
|
|
FRESHNESS=STALE
|
|
LEGACY_SOURCE_PATH=/var/lib/homelab-health/gitea-backup.txt
|
|
CANONICAL_JOB_ID=homelab_appbackup_sqlite_batch
|
|
CANONICAL_STATE=/var/lib/homelab-backup-matrix/state.json
|
|
HEALTH_FILE=homeassistant-backup.txt|mtime=2026-07-09 22:04:04.073386425 +0300|sha256=03ce1462f536f5c3e3eb7b5224465292d1810b063946f859eb6e504eb5bd0a19
|
|
STATUS=OK TS=2026-07-09T19:04:00Z TYPE=homeassistant-backup BAD=0 APP=homeassistant JOB=homelab-pve01-app-homeassistant-20260709T051713Z SIZE=1855722 LOCAL_BACKUP_DELETED=true JSON=/var/lib/homelab-backup/apps/latest-homeassistant.json
|
|
HEALTH_FILE=backup-matrix-desired-state-persistence.txt|mtime=2026-07-13 20:40:37.065354772 +0300|sha256=3d5bdf5d8ec15131c0d801287f324cbe0b6099e1fd079776ce0e3df8a5fa6144
|
|
STATUS=OK TS=2026-07-13T17:40:37Z TYPE=backup-matrix-desired-state-persistence BAD=0 GENERATED_HEAD=af26ec3c4829b1330d49656205ac580ce1706263 GENERATED_PARENT=25568fab24c0594418b3676dc511c8234b4e1ae5 GENERATOR_SHA256=19da7f049b4f7f7286faf292c5419c37cc1e1768c2d8fe36b257ab0cf9c18874 MATRIX_SHA256=7613bcb25d32e61fac114da13af81e38354ec22a012a6deeaa168ee4cc454f87 DESIRED_MATRIX_SHA256=7613bcb25d32e61fac114da13af81e38354ec22a012a6deeaa168ee4cc454f87 MATRIX_STATUS=OK JOBS_BAD=0 FAILED_UNITS=0 BACKUP_RUN=NO RESTORE_RUN=NO CLOUD_WRITE_RUN=NO RETENTION_RUN=NO ROLLBACK=/var/lib/homelab-incidents/backup-matrix-desired-state-persistence-20260713T174036Z
|
|
HEALTH_FILE=alertmanager-two-cloud-backup.txt|mtime=2026-07-10 21:46:36.960914147 +0300|sha256=d90562fac18f643e765e60e54c673839c4ea57e580896241beb0ecc0ea45dea2
|
|
STATUS=OK TS=2026-07-10T18:46:36Z TYPE=alertmanager-two-cloud-backup BAD=0 JOB=homelab-pve01-app-alertmanager-20260710T184605Z ARCHIVE_SIZE_BYTES=1107 LOCAL_STAGING_DELETED=true OFFHOST=OK RESTORE=OK
|
|
HEALTH_FILE=netbird-vps-trust-closure.txt|mtime=2026-07-07 18:29:49.166816011 +0300|sha256=6db8ee1ffbd2c371959917cac5ded56af3ec1829b01bd9c190e3334d6593a4a0
|
|
STATUS=OK TS=2026-07-07T15:29:49Z TYPE=netbird-vps-trust-closure BAD=0 WARN=0 MODE=TRUST_CLOSURE_NO_KNOWN_HOSTS_CHANGE TRUSTED_NETBIRD_IDENTITY=edge-vm TRUSTED_NETBIRD_IP=[IP] PUBLIC_VPS_IP=[IP] PUBLIC_VPS_DECISION=REJECTED_HOSTKEY_MISMATCH PUBLIC_IP_STRICT_BLOCKED=YES PUBLIC_IP_KEYSCAN_MATCH=NO EDGE_REF_OK=1 NETBIRD_PEER_OK=1 REPORT=/var/lib/homelab-netbird-vps-trust/closure-latest.md
|
|
HEALTH_FILE=netbird-vps-trust-clean-seal.txt|mtime=2026-07-07 18:55:43.910666513 +0300|sha256=092eaa2c4f3a5f623e9440135d36e6bae7d8ae46a03d380d14a5e95657812b61
|
|
HEALTH_FILE=searxng-backup.txt|mtime=2026-07-09 12:39:46.404327448 +0300|sha256=c1b03d9b8fee369c584a2c8f725639ed3ca77701d05dbe3b46fe5abb0f37a6b1
|
|
STATUS=OK TS=2026-07-09T09:39:15Z TYPE=searxng-backup JOB=homelab-pve01-app-searxng-20260709T093915Z SOURCE=searxng_stack_config_cache SIZE=738 SHA256=352c9c54437b8ceabc491452d1d7150624eda8c4bf99c28dc78fd488ec06f28c LOCAL_BACKUP_DELETED=true NOTE=service-readiness-backup-proof
|
|
HEALTH_FILE=it-tools-backup.txt|mtime=2026-07-09 22:13:14.698027428 +0300|sha256=1481a6d2014b4caf49b7258d1922b575fadb4645a1a1878457c1f8f3eb8ca525
|
|
STATUS=OK TS=2026-07-09T19:13:00Z TYPE=it-tools-backup BAD=0 APP=it-tools JOB=homelab-pve01-app-it-tools-20260709T191300Z SIZE=689 LOCAL_BACKUP_DELETED=true ENGINE=files JSON=/var/lib/homelab-backup/apps/latest-it-tools.json
|
|
HEALTH_FILE=netbird-vps-offhost.txt|mtime=2026-07-17 04:59:42.647642040 +0300|sha256=04ddca2c2a89ab344726695210540c4ca1358033f2edd22e2c97f1ab0aff4e0b
|
|
STATUS=OK
|
|
SERVICE=netbird-vps-offhost
|
|
TIMESTAMP_UTC=20260717T015941Z
|
|
DST_HOST=[IP]
|
|
DST=/mnt/staging/offhost/netbird-vps-backups-from-pve01
|
|
LOCAL_LATEST=20260701T013054Z
|
|
REMOTE_LATEST=20260701T013054Z
|
|
REMOTE_FILES=30
|
|
REMOTE_SIZE=466M
|
|
HEALTH_FILE=backup-scheduler-fairness-granularity.json|mtime=2026-08-10 20:50:58.416755115 +0300|sha256=5ae14f32dd777dbf4c6dee305db5767e69b9d47edac900d014ad243d8b108fbb
|
|
{
|
|
"status": "OK",
|
|
"root_cause_count": 0,
|
|
"production_changed": true,
|
|
"cloud_changed": false,
|
|
"backup_started": false,
|
|
"applied_at_utc": "2026-08-10T17:50:58Z",
|
|
"target_bytes": true,
|
|
"global_dispatcher": true,
|
|
"fairness_changed": true,
|
|
"granularity_changed": true,
|
|
"one_heavy_target_per_decision": true,
|
|
"heavy_lock_released_between_targets": true,
|
|
"no_progress_guard": true,
|
|
"frozen_dependencies": true,
|
|
"systemd_changed": false,
|
|
"no_immediate_service_start": true,
|
|
"targets": {
|
|
"vm": "b0ca03aff1635a134b1638bf842df65e6fa26d965001b05eeb3ce68bf99811e5",
|
|
"app": "2c52087ea5f25c2763b9126120babdaf3535afff2203bab388ba3a2372a68819",
|
|
"dispatcher": "e83bf96c2e8290f6099e9e63033641fffd983947212ea3392d6197556a199849"
|
|
}
|
|
}
|
|
HEALTH_FILE=p2-media-backup.txt|mtime=2026-07-07 03:57:33.000124656 +0300|sha256=dccd8a9b1c9ae449783cbb8dc16ef9f2ead867b0fec20992722ec5f0aa9a4032
|
|
HEALTH_FILE=actual-budget-backup.txt|mtime=2026-07-09 15:53:45.248005185 +0300|sha256=72d1b31e22e22ac91af66e3d8ae46b20b3b9297809ace524eece22031458fb05
|
|
STATUS=OK TS=2026-07-09T12:53:43Z TYPE=actual-budget-backup JOB=homelab-pve01-app-actual-budget-20260709T051713Z SOURCE=latest-appbackup SIZE=2011 LOCAL_BACKUP_DELETED=true JSON=/var/lib/homelab-backup/apps/latest-actual-budget.json NOTE=service-readiness-backup-proof
|
|
HEALTH_FILE=backup-retention-safe-gc.json|mtime=2026-09-23 03:25:17.780212301 +0300|sha256=d5b48a3782e4181807fa6191ff5de57a1586afc151ec4cc7ba905f6764c03517
|
|
{
|
|
"schema": "homelab-backup-retention-v1",
|
|
"status": "OK",
|
|
"task_id": "backup-retention-safe-gc-v1",
|
|
"change_id": "CR-2026-0033",
|
|
"updated_at_utc": "2026-09-23T00:25:17Z",
|
|
"policy": {
|
|
"vm_keep": 2,
|
|
"app_keep": 7,
|
|
"control_keep": 4,
|
|
"min_delete_age_seconds": 86400,
|
|
"gfs": {
|
|
"enabled": true,
|
|
"critical_vm_count": 8,
|
|
"weekly_completed_buckets": 4,
|
|
"monthly_completed_buckets": 6,
|
|
"completed_periods_only": true
|
|
}
|
|
},
|
|
"owner_aware_gc_proven": true,
|
|
"destructive_canary_proven": true,
|
|
"historical_gc_proven": false,
|
|
"last_plan": {
|
|
"delete_candidates": 4,
|
|
"protected": 0,
|
|
"kept": 139,
|
|
"gfs_kept": 28
|
|
},
|
|
"last_run_deleted": 4,
|
|
"legacy_namespaces_protected": [
|
|
"hb",
|
|
"pve01",
|
|
"control-bootstrap"
|
|
],
|
|
"provider_worm_proven": false
|
|
}
|
|
HEALTH_FILE=backup-matrix-cluster-desired-state-sync.txt|mtime=2026-07-13 20:51:32.416338448 +0300|sha256=c2530de4aaee5f4c2746d4e97b7e6d9ab70c3f4059afc9bacc75fd5e1980d659
|
|
STATUS=OK TS=2026-07-13T17:51:31Z TYPE=backup-matrix-cluster-desired-state-sync BAD=0 PRE_SYNC_HEAD=af26ec3c4829b1330d49656205ac580ce1706263 GENERATED_SYNC_HEAD=597119391e7b265b85fd9a76644e4e33af47e469 GENERATED_SYNC_MODE=NEW_GENERATED_COMMIT GENERATED_SYNC_COMMIT_DELTA=1 MATRIX_STATUS=OK JOBS_TOTAL=35 JOBS_OK=31 JOBS_BAD=0 JOBS_UNMAPPED=0 JOBS_DISABLED=4 CLUSTER_PASSPORT=OK FINAL_READINESS=OK OVERALL=OK CLUSTER_ADMIN=OK PROBE_OK=4 PROBE_BAD=0 FAILED_UNITS=0 DESIRED_STATE_DIRTY=0 MATRIX_SHA256=7613bcb25d32e61fac114da13af81e38354ec22a012a6deeaa168ee4cc454f87 MATRIX_STATE_SHA256=14a31112aac5d577ae01df567921b093b2fedcf7189c4cc8f009216ee0749380 PRESEAL_SHA256=7ea1f710cd311bd51dba05a313b4c540f57d97407f008881e502fc213b4b3c99 BACKUP_RUN=NO RESTORE_RUN=NO CLOUD_WRITE_RUN=NO RETENTION_RUN=NO EVIDENCE_ROOT=/var/lib/homelab-change-sets/backup-matrix-cluster-final-seal/20260713T175131Z
|
|
HEALTH_FILE=control-backup.json|mtime=2026-09-23 11:26:22.485179218 +0300|sha256=0755695105c316d7ee6f3926a1ded277c1c878f92e0691e62eed98f90836dc0b
|
|
{
|
|
"schema": "homelab-control-backup-health-v1",
|
|
"status": "OK",
|
|
"generation": "control-bootstrap-v2-pve01-20260916T120057Z",
|
|
"state_file": "/var/lib/homelab-backup/cloud-quorum/control-bootstrap-v2/control-bootstrap-v2-pve01-20260916T120057Z.json",
|
|
"restore_evidence": "/var/lib/homelab-backup/control-restore-validation/control-bootstrap/control-bootstrap-v2-pve01-20260916T120057Z.json",
|
|
"archive_sha256": "6fa394732cc1cafbf311a022b5fbdb6cf41a6562d9b3503666525c9392503e29",
|
|
"cloud_quorum": 2,
|
|
"control_freshness": "FRESH",
|
|
"full_site_dr_proven": false,
|
|
"updated_at_utc": "2026-09-23T08:26:22Z"
|
|
}
|
|
HEALTH_FILE=homelab-backup-baseline.txt|mtime=2026-06-17 23:02:13.952559036 +0300|sha256=4959757d29bb503315c60468dc0eb080b43cefde23d48e8ab192f6bd93b564d3
|
|
STATUS=OK
|
|
SERVICE=homelab-backup-baseline
|
|
TIMESTAMP_UTC=20260617T200213Z
|
|
ENDPOINT_TOTAL=17
|
|
ENDPOINT_OK=17
|
|
ENDPOINT_BAD=0
|
|
KUMA_TOTAL=18
|
|
KUMA_UP=18
|
|
KUMA_BAD=0
|
|
ENDPOINT_SUMMARY=disk-space.txt:OK;immich-media-backup.txt:OK;immich-media-offhost.txt:OK;memos-backup.txt:OK;memos-offhost.txt:OK;paperless-backup.txt:OK;paperless-offhost.txt:OK;netbird-vps-backup.txt:OK;netbird-vps-offhost.txt:OK;netbird-vps-restore.txt:OK;netbird-peers.txt:OK;filebrowser-backup.txt:OK;filebrowser-offhost.txt:OK;filebrowser-restore.txt:OK;app-stacks-batch-backup.txt:OK;app-stacks-batch-offhost.txt:OK;app-stacks-batch-restore.txt:OK;
|
|
KUMA_MONITORS=1|Homelab Backup Health|active=1|user_id=1|latest_status=1|url=http://[IP]:9101/backup.txt;24|Immich Media Backup|active=1|user_id=1|latest_status=1|url=http://[IP]:9101/immich-media-backup.txt;25|Immich Media Offhost Backup|active=1|user_id=1|latest_status=1|url=http://[IP]:9101/immich-media-offhost.txt;26|Disk Space Health|active=1|user_id=1|latest_status=1|url=http://[IP]:9101/disk-space.txt;27|Memos Backup|active=1|user_id=1|latest_status=1|url=http://[IP]:9101/memos-backup.txt;28|Memos Offhost Backup|active=1|user_id=1|latest_status=1|url=http://[IP]:9101/memos-offhost.txt;29|Paperless Backup|active=1|user_id=1|latest_status=1|url=http://[IP]:9101/paperless-backup.txt;30|Paperless Offhost Backup|active=1|user_id=1|latest_status=1|url=http://[IP]:9101/paperless-offhost.txt;31|NetBird VPS Backup|active=1|user_id=1|latest_status=1|url=http://[IP]:9101/netbird-vps-backup.txt;32|NetBird VPS Offhost Backup|active=1|user_id=1|latest_status=1|url=http://[IP]:9101/netbird-vps-offhost.txt;33|NetBird VPS Restore Validation|active=1|user_id=1|latest_status=1|url=http://[IP]:9101/netbird-vps-restore.txt;34|NetBird Homelab Peers|active=1|user_id=1|latest_status=1|url=http://[IP]:9101/netbird-peers.txt;35|File Browser Backup|active=1|user_id=1|latest_status=1|url=http://[IP]:9101/filebrowser-backup.txt;36|File Browser Offhost Backup|active=1|user_id=1|latest_status=1|url=http://[IP]:9101/filebrowser-offhost.txt;37|File Browser Restore Validation|active=1|user_id=1|latest_status=1|url=http://[IP]:9101/filebrowser-restore.txt;38|App Stacks Batch Backup|active=1|user_id=1|latest_status=1|url=http://[IP]:9101/app-stacks-batch-backup.txt;39|App Stacks Batch Offhost|active=1|user_id=1|latest_status=1|url=http://[IP]:9101/app-stacks-batch-offhost.txt;40|App Stacks Batch Restore|active=1|user_id=1|latest_status=1|url=http://[IP]:9101/app-stacks-batch-restore.txt;
|
|
NOTES=none
|
|
HEALTH_FILE=memos-backup.txt|mtime=2026-08-30 21:08:09.015289529 +0300|sha256=01edf9ba034eb56e7a2ae8abd4ff6ee677fd19c320b4aedefe85d574fc46ee35
|
|
STATUS=FAIL
|
|
TS=2026-08-30T18:08:08Z
|
|
TYPE=backup-matrix-generated
|
|
JOB_ID=memos_backup
|
|
JOB_NAME=memos-backup
|
|
MATRIX_MODE=authoritative
|
|
TIMER=memos-backup.timer
|
|
SERVICE=memos-backup.service
|
|
TIMER_ENABLED=not-found
|
|
TIMER_ACTIVE=inactive
|
|
SOURCE_KIND=health_file
|
|
EVIDENCE_LEVEL=artifact_or_health
|
|
SERVICE_RESULT=success
|
|
SERVICE_EXEC_MAIN_STATUS=0
|
|
SOURCE_HEALTH=/var/lib/homelab-backup/apps/latest-memos.json
|
|
SOURCE_STATUS=OK
|
|
SOURCE_MTIME_UTC=2026-08-06T21:26:13Z
|
|
AGE_SECONDS=2061715
|
|
MAX_AGE_SECONDS=259200
|
|
FRESHNESS=STALE
|
|
LEGACY_SOURCE_PATH=/var/lib/homelab-health/memos-backup.txt
|
|
CANONICAL_JOB_ID=memos_backup
|
|
CANONICAL_STATE=/var/lib/homelab-backup-matrix/state.json
|
|
HEALTH_FILE=netbird-vps-backup.txt|mtime=2026-07-01 04:31:05.433646905 +0300|sha256=372804695d6dba94e29e254f2175701e976692e7b15ee83fdcb6f10e05ca624e
|
|
STATUS=OK
|
|
SERVICE=netbird-vps-backup
|
|
TIMESTAMP_UTC=20260701T013054Z
|
|
VPS_IP=[IP]
|
|
SNAPSHOT=/mnt/staging/netbird-vps-backups/snapshots/20260701T013054Z
|
|
LATEST_SYMLINK=/mnt/staging/netbird-vps-backups/snapshots/latest
|
|
FILES=2
|
|
SIZE=32M
|
|
SHA256=0226dd820cb9ede0ea9e8ea4af6cef656ffa54cfb819048c222a88f4c14d8db9
|
|
HEALTH_FILE=backup-matrix-cluster-preseal-convergence.txt|mtime=2026-07-13 20:48:14.631325305 +0300|sha256=7ea1f710cd311bd51dba05a313b4c540f57d97407f008881e502fc213b4b3c99
|
|
STATUS=OK TS=2026-07-13T17:48:14Z TYPE=backup-matrix-cluster-preseal-convergence BAD=0 MATRIX_STATUS=OK JOBS_TOTAL=35 JOBS_BAD=0 JOBS_UNMAPPED=0 JOBS_DISABLED=4 CLUSTER_PASSPORT=OK FINAL_READINESS=OK OVERALL=OK CLUSTER_ADMIN=OK PROBE_OK=4 PROBE_BAD=0 WEBPANEL=OK FAILED_UNITS=0 DESIRED_STATE_HEAD=af26ec3c4829b1330d49656205ac580ce1706263 DESIRED_STATE_DIRTY=0 MATRIX_SHA256=7613bcb25d32e61fac114da13af81e38354ec22a012a6deeaa168ee4cc454f87 MATRIX_STATE_SHA256=3791fa78cdfb56cd7027f89350dff245abcffc4f935adfc336ceeea00854ac89 CLUSTER_PASSPORT_SHA256=1b8b924ae0e5ce6872fe5671318094ce494806c2db7efe28299b5ff49adc0d0d FINAL_READINESS_SHA256=ac5159fa8c497afe021b5cb232f08cfec0e1a4d516d8aae966ba03f7034f1412 OVERALL_SHA256=6ae529bfdf5811f686d3b53978f453ce4f8757c74fc1be105de219b13a61ba46 BACKUP_RUN=NO RESTORE_RUN=NO CLOUD_WRITE_RUN=NO RETENTION_RUN=NO EVIDENCE_ROOT=/var/lib/homelab-change-sets/backup-matrix-cluster-preseal-resume/20260713T174748Z
|
|
HEALTH_FILE=cluster-admin-stage4f-preapply-backup.txt|mtime=2026-07-14 10:54:50.681548259 +0300|sha256=cc242d1b6e2ef6b6d25f283c7d967bd4985222783ce65050d75e6e61230067ae
|
|
STATUS=OK TS=2026-07-14T07:54:50Z TYPE=cluster-admin-stage4f-preapply-backup-final-seal BAD=0 STAGE4F_CLOSED=YES NEXT_STAGE=4G_PRODUCTION_APPLY_READ_ONLY_ONLY PRODUCTION_CHANGED=NO DATABASE_CHANGED=NO APPLICATION_CHANGED=NO BACKUP_JOB=homelab-pve01-app-cluster-admin-incident-engine-20260714T064327Z ARCHIVE_SIZE_BYTES=2651545 DATABASE_DUMP_SIZE_BYTES=2814735 CURRENT_JOB_VERIFY_MARKERS=2 RESTORE_LIST_CHECK=OK PRODUCTION_STATE=0|0|OK DESIRED_STATE_HEAD=69be75caeecaa242567d2d06c735ac5db3a9c048 DESIRED_STATE_CLEAN=YES REPORT=/var/lib/homelab-cluster-admin-stage4/stage4f-preapply-backup-final-seal-v1/report.txt REPORT_SHA256=9ee244085b9d191370371560690ea87e7bbfe2f14226a06a4e8f6720b3a7439b
|
|
HEALTH_FILE=jellyfin-backup.txt|mtime=2026-07-09 21:16:17.662793576 +0300|sha256=23afe6670bf9250dada540261513ffff2c36ee33963347e5673699d4a10f77a2
|
|
STATUS=OK TS=2026-07-09T18:16:16Z TYPE=jellyfin-backup BAD=0 APP=jellyfin JOB=homelab-pve01-app-jellyfin-20260709T051713Z SIZE=25920 LOCAL_BACKUP_DELETED=true JSON=/var/lib/homelab-backup/apps/latest-jellyfin.json
|
|
HEALTH_FILE=vikunja-backup.txt|mtime=2026-07-09 20:51:20.850795887 +0300|sha256=48bc526e9c24220ad6c15554b85b819d8b6ba35d854ab87da2324b1534e194c6
|
|
STATUS=OK TS=2026-07-09T17:51:14Z TYPE=vikunja-backup BAD=0 APP=vikunja JOB=homelab-pve01-app-vikunja-20260709T051713Z SIZE=34831 LOCAL_BACKUP_DELETED=true JSON=/var/lib/homelab-backup/apps/latest-vikunja.json
|
|
HEALTH_FILE=hapusya-site-v2-preapply-backup.txt|mtime=2026-07-13 14:34:59.208365351 +0300|sha256=506879fe26ddf3c392c4d485d1db145cde846336cd626096555e2021ab2705af
|
|
STATUS=OK TS=2026-07-13T11:34:59Z TYPE=hapusya-site-v2-preapply-backup BAD=0 NPMPLUS_JOB=homelab-pve01-app-npmplus-edge-20260713T113435Z TWO_CLOUDS=OK RESTORE_CHECK=OK ROLLBACK=/var/lib/homelab-incidents/hapusya-site-v2-preapply-20260713T113458Z ROLLBACK_MANIFEST_SHA256=144506c766fe9fd3152e4edf94a8867895b3dc54f29faf1e329773922b8c5640 CANDIDATE_TREE_SHA256=0eb098f91ea706c884d36a8ce806968e82e715ded66bb96f61c2ed8106faca93 CANDIDATE_BROWSER=OK BROWSER_RUNS=5 VIDEO_DESKTOP=OK VIDEO_MOBILE=OK STATIC_FALLBACK=OK PRODUCTION_CHANGED=NO
|
|
HEALTH_FILE=ntfy-backup.txt|mtime=2026-07-09 23:39:41.327493875 +0300|sha256=e28d674bd0ba4c088c6c9f523da493e3674750837ab417223f7405ea61d8b020
|
|
STATUS=OK TS=2026-07-09T20:39:29Z TYPE=ntfy-backup BAD=0 APP=ntfy JOB=homelab-pve01-app-ntfy-20260709T051713Z SIZE=13516 LOCAL_BACKUP_DELETED=true JSON=/var/lib/homelab-backup/apps/latest-ntfy.json
|
|
HEALTH_FILE=immich-media-backup.txt|mtime=2026-08-30 21:08:08.994511799 +0300|sha256=f94d804693a2ca18e51c66ade993a95c9b8ae183730e31013e3886010b03ff63
|
|
STATUS=FAIL
|
|
TS=2026-08-30T18:08:08Z
|
|
TYPE=backup-matrix-generated
|
|
JOB_ID=immich_media_backup
|
|
JOB_NAME=immich-media-offhost-sync
|
|
MATRIX_MODE=authoritative
|
|
TIMER=immich-media-offhost-sync.timer
|
|
SERVICE=immich-media-offhost-sync.service
|
|
TIMER_ENABLED=not-found
|
|
TIMER_ACTIVE=inactive
|
|
SOURCE_KIND=health_file
|
|
EVIDENCE_LEVEL=artifact_or_health
|
|
SERVICE_RESULT=success
|
|
SERVICE_EXEC_MAIN_STATUS=0
|
|
SOURCE_HEALTH=/var/lib/homelab-health/immich-media-offhost.txt
|
|
SOURCE_STATUS=OK
|
|
SOURCE_MTIME_UTC=2026-08-08T05:52:15Z
|
|
AGE_SECONDS=1944953
|
|
MAX_AGE_SECONDS=259200
|
|
FRESHNESS=STALE
|
|
LEGACY_SOURCE_PATH=/var/lib/homelab-health/immich-media-backup.txt
|
|
CANONICAL_JOB_ID=immich_media_backup
|
|
CANONICAL_STATE=/var/lib/homelab-backup-matrix/state.json
|
|
HEALTH_FILE=paperless-backup.txt|mtime=2026-07-18 01:52:29.893166209 +0300|sha256=8ae7e31227e48c86412eb9e49fa91ce3cbb97454ea3414e25d10ef5432e6cd46
|
|
STATUS=OK
|
|
TS=2026-07-17T22:52:29Z
|
|
TYPE=backup-matrix-generated
|
|
JOB_ID=paperless_backup
|
|
JOB_NAME=paperless-backup
|
|
MATRIX_MODE=authoritative
|
|
TIMER=paperless-backup.timer
|
|
SERVICE=paperless-backup.service
|
|
TIMER_ENABLED=enabled
|
|
TIMER_ACTIVE=active
|
|
SOURCE_KIND=health_file
|
|
EVIDENCE_LEVEL=artifact_or_health
|
|
SERVICE_RESULT=success
|
|
SERVICE_EXEC_MAIN_STATUS=0
|
|
SOURCE_HEALTH=/var/lib/homelab-backup/apps/latest-paperless.json
|
|
SOURCE_STATUS=OK
|
|
SOURCE_MTIME_UTC=2026-07-17T02:30:25Z
|
|
AGE_SECONDS=73324
|
|
MAX_AGE_SECONDS=259200
|
|
FRESHNESS=FRESH
|
|
LEGACY_SOURCE_PATH=/var/lib/homelab-health/paperless-backup.txt
|
|
CANONICAL_JOB_ID=paperless_backup
|
|
CANONICAL_STATE=/var/lib/homelab-backup-matrix/state.json
|
|
HEALTH_FILE=cluster-admin-stage1-preapply-backup.txt|mtime=2026-07-13 03:11:06.700511934 +0300|sha256=a70d7bba3c2b50deaa117e56862659b857e72682ec829ed7724b03dc92111105
|
|
STATUS=OK TS=2026-07-13T00:10:57Z TYPE=cluster-admin-stage1-preapply-backup BAD=0 PRODUCTION_CHANGED=BACKUP_ARTIFACTS_ONLY PRODUCTION_DATABASE_CHANGED=NO SERVICE_RESTARTED=NO CLOUD_WRITE_RUN=NO ROLLBACK_POINT=VERIFIED DIRECT_DB_SNAPSHOT=VERIFIED DIRECT_DB_DUMP_SIZE_BYTES=1710825 DIRECT_DB_DUMP_SHA256=c0f13cec235ab3242eb8da4cad12962df1a2e37b25792a8ce9944c1d7de4312a APPBACKUP_JOB=homelab-pve01-app-cluster-admin-incident-engine-20260713T000543Z APPBACKUP_STATUS=success APPBACKUP_RESTORE_CHECK=OK MIGRATION_DRY_RUN=OK IDEMPOTENCE_TEST=OK ROLLBACK_TEST=OK COLLECTOR_COMPATIBILITY_TRIGGER=OK PRODUCTION_SCHEMA_UNCHANGED=YES READY_FOR_PRODUCTION_APPLY=YES DESIRED_STATE_HEAD=1379b4fb4e809e56e9bc16f0e0b19b0a509192d5 PREAPPLY_ROOT=/var/lib/homelab-stage1-preapply/20260713T000041Z REMOTE_SNAPSHOT_DIR=/var/lib/cluster-admin-preapply/20260713T000041Z EVIDENCE_MANIFEST=/var/lib/homelab-stage1-preapply/20260713T000041Z/SHA256SUMS
|
|
HEALTH_FILE=cadvisor-backup.txt|mtime=2026-07-09 11:25:57.266067724 +0300|sha256=cb0bcacf449ab29e8954014fb689d1022527f0b67aae0d4f0c808358cae55e07
|
|
STATUS=OK TS=2026-07-09T08:25:56Z TYPE=cadvisor-backup JOB=homelab-pve01-app-cadvisor-20260709T082556Z SOURCE=cadvisor_stack_proxy SIZE=1384 SHA256=33e419912291daab19f8039dfe8971f277be0fe577eb3d0e538263bdf918eac8 LOCAL_BACKUP_DELETED=true NOTE=service-readiness-backup-proof
|
|
HEALTH_FILE=dockge-stacks-backup.txt|mtime=2026-07-09 13:12:46.609379653 +0300|sha256=f20ebfd8e230858d31286670ccb6a4699d4251209765fb37e3ff4309435113f5
|
|
STATUS=OK TS=2026-07-09T10:12:44Z TYPE=dockge-stacks-backup JOB=homelab-pve01-app-dockge-stacks-20260709T072218Z SOURCE=latest-appbackup SIZE=134913 LOCAL_BACKUP_DELETED=true JSON=/var/lib/homelab-backup/apps/latest-dockge-stacks.json NOTE=service-readiness-backup-proof
|
|
HEALTH_FILE=dockge-backup.txt|mtime=2026-07-09 13:12:46.609379653 +0300|sha256=3f5198e9d0eb7852e4ee563109c801be03be268b2b10afeca68fbc39be4d35f7
|
|
STATUS=OK TS=2026-07-09T10:12:44Z TYPE=dockge-backup JOB=homelab-pve01-app-dockge-stacks-20260709T072218Z SOURCE=appbackup-dockge-stacks SIZE=134913 LOCAL_BACKUP_DELETED=true JSON=/var/lib/homelab-backup/apps/latest-dockge.json NOTE=service-readiness-backup-proof
|
|
HEALTH_FILE=backup-lock-fd-hardening.json|mtime=2026-08-10 17:49:25.372646496 +0300|sha256=d64d18259e58525a9549f6d2b40b331ea6cc7b7057fd17050aebc585d73731a5
|
|
{
|
|
"status": "OK",
|
|
"root_cause_count": 0,
|
|
"production_changed": true,
|
|
"cloud_changed": false,
|
|
"backup_started": false,
|
|
"applied_at_utc": "2026-08-10T14:49:25Z",
|
|
"target_bytes": true,
|
|
"fd_child_boundary": true,
|
|
"parent_lock_ownership_preserved": true,
|
|
"systemd_changed": false,
|
|
"no_immediate_service_start": true,
|
|
"targets": {
|
|
"vm": "55c2c30d93590ce0367667c2de4cf8e6436bd10a2a35fb7b775f918ba9b52ffb",
|
|
"app": "adcd592c619c96c99be19777eff98bbf0226e878a9635c718e9113b662960a5a"
|
|
}
|
|
}
|
|
HEALTH_FILE=backup-sla.txt|mtime=2026-08-30 21:08:08.601604759 +0300|sha256=4762bb7cfb98ab1311adbd1e80287f7e18f79983656a888a056ed730ecdf0372
|
|
STATUS=FAIL
|
|
TS=2026-08-30T18:08:08Z
|
|
TYPE=backup-matrix-generated
|
|
JOB_ID=homelab_backup_sla_health
|
|
JOB_NAME=homelab-backup-sla-health
|
|
MATRIX_MODE=authoritative
|
|
TIMER=homelab-backup-sla-health.timer
|
|
SERVICE=homelab-backup-sla-health.service
|
|
TIMER_ENABLED=not-found
|
|
TIMER_ACTIVE=inactive
|
|
SOURCE_KIND=health_file
|
|
EVIDENCE_LEVEL=artifact_or_health
|
|
SERVICE_RESULT=success
|
|
SERVICE_EXEC_MAIN_STATUS=0
|
|
SOURCE_HEALTH=/var/lib/homelab-backup-matrix/evidence/homelab_backup_sla_health/0-backup-sla.txt
|
|
SOURCE_STATUS=WARN
|
|
SOURCE_MTIME_UTC=2026-08-06T21:34:03Z
|
|
AGE_SECONDS=2061245
|
|
MAX_AGE_SECONDS=172800
|
|
FRESHNESS=STALE
|
|
LEGACY_SOURCE_PATH=/var/lib/homelab-health/backup-sla.txt
|
|
CANONICAL_JOB_ID=homelab_backup_sla_health
|
|
CANONICAL_STATE=/var/lib/homelab-backup-matrix/state.json
|
|
HEALTH_FILE=deep-service-backup-audit.txt|mtime=2026-07-10 00:12:45.326483756 +0300|sha256=16dfd49e8f0618a836c473874b7a70fa896771dff5e2613e510e6d0a02eca7b4
|
|
STATUS=OK TS=2026-07-09T21:12:44Z TYPE=deep-service-backup-audit BAD=0 SERVICE_READINESS_OK_COUNT=47 NON_OK=0 APPBACKUP_FLAG_GAPS=0 CONTAINER_GAPS=0 PROM_FIRING=0 BACKUP_SLA=OK ALERTING=OK EXTERNAL_LINKS_CLASSIFIED=OK SERVICES_REMAINING_TOTAL=0 BACKUP_GAPS_TOTAL=0 COMMANDS_REMAINING_TOTAL=0
|
|
HEALTH_FILES_END=1
|
|
DEPENDENCY_FILE_INDEX_BEGIN=1
|
|
DEPENDENCY_FILE_COUNT=34
|
|
DEPENDENCY_FILE|path=/etc/homelab-backup-v2/targets.json|mtime=2026-09-22 12:19:09.941039348 +0300|sha256=decea6b876c8ede15c91cad69c8e15e46021e5e96d25d415139bf2c524e23f7b
|
|
2: "created_by": "SUPPORT-260914-INFRA-DR-BACKUPV2-SHADOW-PERSISTENT-PRODUCERS-STAGE-GREEN1-389R1",
|
|
169: "logical_id": "md-relay",
|
|
193: "logical_id": "us-netbird",
|
|
DEPENDENCY_FILE|path=/etc/systemd/system/netbird-vps-backup.service.d/10-superseded-noop.conf|mtime=2026-07-08 08:25:22.050610124 +0300|sha256=4cf3f2825b770b32447624a611bce605d97987bd2c6e8862c84d35b166f9102a
|
|
3:ExecStart=/usr/local/sbin/homelab-superseded-unit-ok netbird-vps-backup.service superseded_by_current_appbackup_or_trust_proof
|
|
DEPENDENCY_FILE|path=/root/771ad_final_crowdsec_capi_netbird_closure.sh|mtime=2026-07-02 22:10:57.781648619 +0300|sha256=c99d08385f5d7c0a266c1c8002b3c7b054e1bb561eb48b1f8a227f37fe430ab6
|
|
8:PROOF="/root/evidence/771AD_FINAL_CROWDSEC_CAPI_NETBIRD_CLOSURE_${TS}_PROOF.txt"
|
|
24: echo "STEP=771AD_FINAL_CROWDSEC_CAPI_NETBIRD_CLOSURE"
|
|
47: echo NETBIRD_EXIT_ROUTE_VERIFY
|
|
48: netbird status 2>&1 | redact || true
|
|
59: curl -4 -sk --http1.1 --connect-timeout 12 --max-time 45 -o /tmp/771ad_capi.body -w \"CAPI_HTTP=%{http_code}\\n\" https://api.crowdsec.net/v3/watchers/login || true
|
|
61: echo CROWDSEC_STATUS
|
|
62: docker ps --format \"{{.Names}} {{.Status}} {{.Ports}}\" | grep -E \"^(crowdsec|socket-proxy|npmplus)\" || true
|
|
63: docker exec crowdsec cscli lapi status 2>&1 | redact || true
|
|
64: docker exec crowdsec cscli capi status 2>&1 | redact || true
|
|
67: cd /opt/stacks/crowdsec
|
|
78: docker logs --since 10m crowdsec 2>&1 | grep -Ei \"fatal|TLS handshake timeout|unable to run local API|failed to get docker info|lookup socket-proxy|server misbehaving|mapping value is not allowed|missing credentials field|online_api_credentials\" | redact || echo RECENT_FATALS_NONE
|
|
80: echo HEALTH_FILE
|
|
81: cat /var/lib/homelab-health/crowdsec-capi.txt 2>/dev/null | redact || true
|
|
97: if ! grep -q 'CROWDSEC_CAPI_NETBIRD_EXIT_ROUTE_20260702' "$REF"; then
|
|
100:## CROWDSEC_CAPI_NETBIRD_EXIT_ROUTE_20260702
|
|
101:- edge-vm CrowdSec CAPI is registered and enabled through NetBird exit route.
|
|
102:- Client peer: edge-vm.netbird.selfhosted / [IP].
|
|
103:- Primary exit routing peer: relay.netbird.selfhosted / [IP] / Moldova.
|
|
104:- Backup egress peer present: mail.netbird.selfhosted / [IP] / USA.
|
|
105:- NetBird exit route proof on edge-vm: wg allowed-ips includes [IP]/0 via relay; ip route get [IP] uses wt0 table 7120.
|
|
107:- CrowdSec proof: cscli lapi status OK, cscli capi status OK, CAPI sharing/blocklist pull enabled, no DISABLE_ONLINE_API or -no-capi flags in active compose.
|
|
109:- Unauthenticated https://api.crowdsec.net/v3/watchers/login returning HTTP 403 is expected network reachability proof.
|
|
118: if ! grep -q 'LESSON_20260702_CROWDSEC_NETBIRD_EXIT_ROUTE' "$ERR"; then
|
|
121:## LESSON_20260702_CROWDSEC_NETBIRD_EXIT_ROUTE
|
|
122:- Do not treat e3qxxx.netbird.selfhosted / [IP] and e3qxxx-183-106.netbird.selfhosted / [IP] as VPS egress nodes; they are user mobile peers.
|
|
123:- Correct egress candidates are relay.pvepro.ru / relay.netbird.selfhosted / [IP] for Moldova and mail.pvepro.ru / mail.netbird.selfhosted / [IP] for USA.
|
|
124:- Access Policy edge-vm -> relay/mail only proves peer reachability. It is not an internet exit route.
|
|
125:- Required NetBird fix for CrowdSec CAPI was Network Routing exit route [IP]/0 distributed to edge-vm group, routing peer relay, masquerade/NAT enabled.
|
|
126:- Before attempting CrowdSec CAPI, verify on edge-vm: ip route get [IP] uses wt0, wg allowed-ips contains [IP]/0, and Cloudflare trace no longer shows home IP [IP].
|
|
134: echo "STATUS=OK_771AD_FINAL_CROWDSEC_CAPI_NETBIRD_CLOSURE_DONE"
|
|
DEPENDENCY_FILE|path=/root/771ag_publish_final_health_index.sh|mtime=2026-07-02 22:30:48.251143514 +0300|sha256=8b7c814c3893ebb2068d90c29ec01460895aba7cba44d78ba8d7d2a8936c1b7a
|
|
8:PROOF="/root/evidence/771AG_PUBLISH_FINAL_HEALTH_INDEX_${TS}_PROOF.txt"
|
|
10:LOCAL_HEALTH="/var/lib/homelab-health/crowdsec-capi-netbird-final.txt"
|
|
23: echo "STEP=771AG_PUBLISH_FINAL_HEALTH_INDEX"
|
|
26: echo "CONFIG_CHANGE=NO_SERVICE_CHANGE_HEALTH_INDEX_ONLY"
|
|
31: grep -n 'CROWDSEC_CAPI_NETBIRD_EXIT_ROUTE_20260702' /etc/pve/31_HOMELAB_REFERENCE.md || true
|
|
32: grep -n 'LESSON_20260702_CROWDSEC_NETBIRD_EXIT_ROUTE' /etc/pve/HOMELAB_ASSISTANT_ERROR_REGISTER.md || true
|
|
34: echo "WRITE_LOCAL_HEALTH_SUMMARY"
|
|
35: mkdir -p /var/lib/homelab-health
|
|
36: cat >"$LOCAL_HEALTH" <<EOF
|
|
39:CHECK=crowdsec-capi-netbird-final
|
|
40:DETAIL=CrowdSec CAPI/LAPI stable through NetBird exit route relay [IP]; WARP absent; route smoke OK; documentation and error register updated.
|
|
43:EDGE_NETBIRD_IP=[IP]
|
|
44:PRIMARY_EXIT=relay.netbird.selfhosted
|
|
45:PRIMARY_EXIT_NETBIRD_IP=[IP]
|
|
46:BACKUP_EXIT=mail.netbird.selfhosted
|
|
47:BACKUP_EXIT_NETBIRD_IP=[IP]
|
|
50:PROOFS=771AC_REGISTER_CROWDSEC_CAPI_AFTER_EXIT_ROUTE,771AD_FINAL_CROWDSEC_CAPI_NETBIRD_CLOSURE,771AE_CROWDSEC_CAPI_SOAK_FINAL,771AF_CROWDSEC_CAPI_PROOF_INDEX
|
|
53: chmod 0644 "$LOCAL_HEALTH"
|
|
54: cat "$LOCAL_HEALTH"
|
|
67: echo EDGE_HEALTH_FILE
|
|
68: cat /var/lib/homelab-health/crowdsec-capi.txt 2>/dev/null | redact || true
|
|
77: echo EDGE_CROWDSEC
|
|
78: docker exec crowdsec cscli lapi status 2>&1 | redact || true
|
|
79: docker exec crowdsec cscli capi status 2>&1 | redact || true
|
|
82: docker inspect crowdsec --format \"RestartCount={{.RestartCount}}\" 2>/dev/null || true
|
|
83: cd /opt/stacks/crowdsec
|
|
93: for cmd in /usr/local/sbin/homelab-health-metrics /usr/local/sbin/homelab-evidence-catalog.sh; do
|
|
111: echo "LOCAL_HEALTH_FINAL"
|
|
112: cat "$LOCAL_HEALTH"
|
|
115: echo "STATUS=OK_771AG_PUBLISH_FINAL_HEALTH_INDEX_DONE"
|
|
DEPENDENCY_FILE|path=/root/771ak_final_done_marker.sh|mtime=2026-07-02 22:34:56.049721292 +0300|sha256=64f3b7e547623a9908eab9f09a1edcaeac8795af30db85e975dfa2e95cbe0d2a
|
|
15: echo "RESULT=CROWDSEC_CAPI_CLOSED"
|
|
18: echo "EXIT_ROUTE=relay.netbird.selfhosted/[IP]"
|
|
23: echo "REFERENCE_MARKER_LINE=$(grep -n 'CROWDSEC_CAPI_NETBIRD_EXIT_ROUTE_20260702' /etc/pve/31_HOMELAB_REFERENCE.md | cut -d: -f1 | head -1)"
|
|
24: echo "ERROR_REGISTER_MARKER_LINE=$(grep -n 'LESSON_20260702_CROWDSEC_NETBIRD_EXIT_ROUTE' /etc/pve/HOMELAB_ASSISTANT_ERROR_REGISTER.md | cut -d: -f1 | head -1)"
|
|
25: echo "FINAL_HEALTH=$(grep -m1 '^STATUS=' /var/lib/homelab-health/crowdsec-capi-netbird-final.txt 2>/dev/null || echo MISSING)"
|
|
26: echo "SOAK_HEALTH_ON_EDGE=$(ssh debian@[IP] 'sudo grep -m1 "^STATUS=" /var/lib/homelab-health/crowdsec-capi.txt 2>/dev/null || echo MISSING')"
|
|
28: ls -1 /root/evidence/771AC_REGISTER_CROWDSEC_CAPI_AFTER_EXIT_ROUTE_20260702T190428Z_PROOF.txt \
|
|
29: /root/evidence/771AD_FINAL_CROWDSEC_CAPI_NETBIRD_CLOSURE_20260702T191057Z_PROOF.txt \
|
|
30: /root/evidence/771AE_CROWDSEC_CAPI_SOAK_FINAL_20260702T191745Z_PROOF.txt \
|
|
31: /root/evidence/771AF_CROWDSEC_CAPI_PROOF_INDEX_20260702T192904Z.txt \
|
|
32: /root/evidence/771AG_PUBLISH_FINAL_HEALTH_INDEX_20260702T193048Z_PROOF.txt \
|
|
DEPENDENCY_FILE|path=/root/771l_crowdsec_capi_netbird_vps_egress_remote_20260702T151045Z.sh|mtime=2026-07-02 18:10:45.300196691 +0300|sha256=6a6df3cc085363c2bbbb92b8c3083123fb0e583e14e242c583a321bdb5612002
|
|
5:STACK=/opt/stacks/crowdsec
|
|
7:BACKUP_DIR="$STACK/manual-backups/771l-netbird-vps-egress-$TS"
|
|
8:HEALTH_DIR=/var/lib/homelab-health
|
|
9:HEALTH_FILE="$HEALTH_DIR/crowdsec-capi.txt"
|
|
10:EGRESS_DIR=/etc/homelab-crowdsec-capi-netbird-egress
|
|
11:EGRESS_SERVICE=homelab-crowdsec-capi-netbird-egress.service
|
|
14:mkdir -p "$BACKUP_DIR" "$HEALTH_DIR" "$EGRESS_DIR/profiles"
|
|
20:write_health() {
|
|
26: echo "CHECK=crowdsec-capi"
|
|
29: } > "$HEALTH_FILE"
|
|
33: docker network inspect crowdsec_default --format '{{(index .IPAM.Config 0).Gateway}}' 2>/dev/null || echo "[IP]"
|
|
41: crowdsec:
|
|
66: docker compose up -d --force-recreate crowdsec
|
|
74: state="$(docker inspect crowdsec --format 'Running={{.State.Running}} Restarting={{.State.Restarting}} RestartCount={{.RestartCount}} StartedAt={{.State.StartedAt}} ExitCode={{.State.ExitCode}}' 2>/dev/null || true)"
|
|
75: health="$(curl -sk --connect-timeout 3 --max-time 6 -o /tmp/771l_health.html -w '%{http_code}' http://[IP]:8088/health || true)"
|
|
78: fatal="$(docker logs --since 30s crowdsec 2>&1 | grep -Eic 'fatal|unable to run local API|failed to get docker info|lookup socket-proxy|server misbehaving|mapping value is not allowed' || true)"
|
|
79: echo "LAPI_TRY=$n STATE=$state HEALTH=$health LAPI_RC=${lapi_rc:-NA} FATAL_COUNT_30S=$fatal"
|
|
83: && test "$health" = "200" \
|
|
126:remove("/opt/stacks/crowdsec/config/config.yaml")
|
|
127:remove("/opt/stacks/crowdsec/config/user.yaml")
|
|
134:p = Path("/opt/stacks/crowdsec/config/config.yaml")
|
|
165: " credentials_path: /etc/crowdsec/online_api_credentials.yaml",
|
|
177: crowdsec:
|
|
182: NO_PROXY: "[IP],localhost,[IP],[IPV6],socket-proxy,crowdsec"
|
|
221:collect_netbird_peers() {
|
|
222: echo "NETBIRD_STATUS"
|
|
223: if ! command -v netbird >/dev/null 2>&1; then
|
|
224: echo "NETBIRD_CLI_MISSING"
|
|
228: netbird status 2>&1 | redact | tee /tmp/771l_netbird_status.txt || true
|
|
229: netbird status --json >/tmp/771l_netbird_status.json 2>/tmp/771l_netbird_status_json.err || true
|
|
230: netbird ip 2>/tmp/771l_netbird_ip.err | tee /tmp/771l_netbird_ip.txt || true
|
|
232: echo "NETBIRD_INTERFACES"
|
|
233: ip -o -4 addr show | grep -Ei 'wt|netbird|wg|100\.' || true
|
|
235: python3 - <<'PY' >/tmp/771l_netbird_peers.tsv
|
|
240:for p in [Path("/tmp/771l_netbird_ip.txt")]:
|
|
244:text = Path("/tmp/771l_netbird_status.txt").read_text(errors="ignore") if Path("/tmp/771l_netbird_status.txt").exists() else ""
|
|
DEPENDENCY_FILE|path=/root/771l_crowdsec_capi_netbird_vps_egress.sh|mtime=2026-07-02 18:10:45.295356498 +0300|sha256=401396a25d4a5cfa487f67b355b45b27140ed1ac6b49b41e342306cb618dfdb5
|
|
8:PROOF="/root/evidence/771L_CROWDSEC_CAPI_NETBIRD_VPS_EGRESS_${TS}_PROOF.txt"
|
|
10:REMOTE_SCRIPT="/tmp/771l_crowdsec_capi_netbird_vps_egress_${TS}.sh"
|
|
11:LOCAL_REMOTE_SCRIPT="/root/771l_crowdsec_capi_netbird_vps_egress_remote_${TS}.sh"
|
|
18:STACK=/opt/stacks/crowdsec
|
|
20:BACKUP_DIR="$STACK/manual-backups/771l-netbird-vps-egress-$TS"
|
|
21:HEALTH_DIR=/var/lib/homelab-health
|
|
22:HEALTH_FILE="$HEALTH_DIR/crowdsec-capi.txt"
|
|
23:EGRESS_DIR=/etc/homelab-crowdsec-capi-netbird-egress
|
|
24:EGRESS_SERVICE=homelab-crowdsec-capi-netbird-egress.service
|
|
27:mkdir -p "$BACKUP_DIR" "$HEALTH_DIR" "$EGRESS_DIR/profiles"
|
|
33:write_health() {
|
|
39: echo "CHECK=crowdsec-capi"
|
|
42: } > "$HEALTH_FILE"
|
|
46: docker network inspect crowdsec_default --format '{{(index .IPAM.Config 0).Gateway}}' 2>/dev/null || echo "[IP]"
|
|
54: crowdsec:
|
|
79: docker compose up -d --force-recreate crowdsec
|
|
87: state="$(docker inspect crowdsec --format 'Running={{.State.Running}} Restarting={{.State.Restarting}} RestartCount={{.RestartCount}} StartedAt={{.State.StartedAt}} ExitCode={{.State.ExitCode}}' 2>/dev/null || true)"
|
|
88: health="$(curl -sk --connect-timeout 3 --max-time 6 -o /tmp/771l_health.html -w '%{http_code}' http://[IP]:8088/health || true)"
|
|
91: fatal="$(docker logs --since 30s crowdsec 2>&1 | grep -Eic 'fatal|unable to run local API|failed to get docker info|lookup socket-proxy|server misbehaving|mapping value is not allowed' || true)"
|
|
92: echo "LAPI_TRY=$n STATE=$state HEALTH=$health LAPI_RC=${lapi_rc:-NA} FATAL_COUNT_30S=$fatal"
|
|
96: && test "$health" = "200" \
|
|
139:remove("/opt/stacks/crowdsec/config/config.yaml")
|
|
140:remove("/opt/stacks/crowdsec/config/user.yaml")
|
|
147:p = Path("/opt/stacks/crowdsec/config/config.yaml")
|
|
178: " credentials_path: /etc/crowdsec/online_api_credentials.yaml",
|
|
190: crowdsec:
|
|
195: NO_PROXY: "[IP],localhost,[IP],[IPV6],socket-proxy,crowdsec"
|
|
234:collect_netbird_peers() {
|
|
235: echo "NETBIRD_STATUS"
|
|
236: if ! command -v netbird >/dev/null 2>&1; then
|
|
237: echo "NETBIRD_CLI_MISSING"
|
|
241: netbird status 2>&1 | redact | tee /tmp/771l_netbird_status.txt || true
|
|
242: netbird status --json >/tmp/771l_netbird_status.json 2>/tmp/771l_netbird_status_json.err || true
|
|
243: netbird ip 2>/tmp/771l_netbird_ip.err | tee /tmp/771l_netbird_ip.txt || true
|
|
245: echo "NETBIRD_INTERFACES"
|
|
246: ip -o -4 addr show | grep -Ei 'wt|netbird|wg|100\.' || true
|
|
DEPENDENCY_FILE|path=/root/771m_purge_warp_fix_netbird_egress.sh|mtime=2026-07-02 18:16:37.903239779 +0300|sha256=0ae3b0e1e8016da55cf1756e868c51a23fe12328fae6ef4baa61c50b9e48e715
|
|
8:PROOF="/root/evidence/771M_PURGE_WARP_FIX_NETBIRD_EGRESS_${TS}_PROOF.txt"
|
|
10:REMOTE_SCRIPT="/tmp/771m_purge_warp_fix_netbird_${TS}.sh"
|
|
11:LOCAL_REMOTE_SCRIPT="/root/771m_purge_warp_fix_netbird_remote_${TS}.sh"
|
|
18:STACK=/opt/stacks/crowdsec
|
|
20:BACKUP_DIR="$STACK/manual-backups/771m-purge-warp-netbird-egress-$TS"
|
|
21:HEALTH_DIR=/var/lib/homelab-health
|
|
22:HEALTH_FILE="$HEALTH_DIR/crowdsec-capi.txt"
|
|
23:EGRESS_DIR=/etc/homelab-crowdsec-capi-netbird-egress
|
|
24:EGRESS_SERVICE=homelab-crowdsec-capi-netbird-egress.service
|
|
28:mkdir -p "$BACKUP_DIR" "$HEALTH_DIR" "$EGRESS_DIR/profiles"
|
|
34:write_health() {
|
|
40: echo "CHECK=crowdsec-capi"
|
|
43: } > "$HEALTH_FILE"
|
|
47: docker network inspect crowdsec_default --format '{{(index .IPAM.Config 0).Gateway}}' 2>/dev/null || echo "[IP]"
|
|
55: crowdsec:
|
|
80: docker compose up -d --force-recreate crowdsec
|
|
88: state="$(docker inspect crowdsec --format 'Running={{.State.Running}} Restarting={{.State.Restarting}} RestartCount={{.RestartCount}} StartedAt={{.State.StartedAt}} ExitCode={{.State.ExitCode}}' 2>/dev/null || true)"
|
|
89: health="$(curl -sk --connect-timeout 3 --max-time 6 -o /tmp/771m_health.html -w '%{http_code}' http://[IP]:8088/health || true)"
|
|
92: fatal="$(docker logs --since 30s crowdsec 2>&1 | grep -Eic 'fatal|unable to run local API|failed to get docker info|lookup socket-proxy|server misbehaving|mapping value is not allowed' || true)"
|
|
93: echo "LAPI_TRY=$n STATE=$state HEALTH=$health LAPI_RC=${lapi_rc:-NA} FATAL_COUNT_30S=$fatal"
|
|
97: && test "$health" = "200" \
|
|
140:remove("/opt/stacks/crowdsec/config/config.yaml")
|
|
141:remove("/opt/stacks/crowdsec/config/user.yaml")
|
|
148:p = Path("/opt/stacks/crowdsec/config/config.yaml")
|
|
179: " credentials_path: /etc/crowdsec/online_api_credentials.yaml",
|
|
191: crowdsec:
|
|
196: NO_PROXY: "[IP],localhost,[IP],[IPV6],socket-proxy,crowdsec"
|
|
258:collect_netbird_peers() {
|
|
259: echo "NETBIRD_STATUS"
|
|
260: if ! command -v netbird >/dev/null 2>&1; then
|
|
261: echo "NETBIRD_CLI_MISSING"
|
|
265: netbird status 2>&1 | redact | tee /tmp/771m_netbird_status.txt || true
|
|
266: netbird status --json >/tmp/771m_netbird_status.json 2>/tmp/771m_netbird_status_json.err || true
|
|
267: netbird ip 2>/tmp/771m_netbird_ip.err | tee /tmp/771m_netbird_ip.txt || true
|
|
269: echo "NETBIRD_INTERFACES"
|
|
270: ip -o -4 addr show | grep -Ei 'wt|netbird|wg|100\.' || true
|
|
DEPENDENCY_FILE|path=/root/771m_purge_warp_fix_netbird_remote_20260702T151637Z.sh|mtime=2026-07-02 18:16:37.909445276 +0300|sha256=941f683d148a01d9af9ef6ce938c8ee6874d7af1ac1ac5ee96c7d9ca7cc262a4
|
|
5:STACK=/opt/stacks/crowdsec
|
|
7:BACKUP_DIR="$STACK/manual-backups/771m-purge-warp-netbird-egress-$TS"
|
|
8:HEALTH_DIR=/var/lib/homelab-health
|
|
9:HEALTH_FILE="$HEALTH_DIR/crowdsec-capi.txt"
|
|
10:EGRESS_DIR=/etc/homelab-crowdsec-capi-netbird-egress
|
|
11:EGRESS_SERVICE=homelab-crowdsec-capi-netbird-egress.service
|
|
15:mkdir -p "$BACKUP_DIR" "$HEALTH_DIR" "$EGRESS_DIR/profiles"
|
|
21:write_health() {
|
|
27: echo "CHECK=crowdsec-capi"
|
|
30: } > "$HEALTH_FILE"
|
|
34: docker network inspect crowdsec_default --format '{{(index .IPAM.Config 0).Gateway}}' 2>/dev/null || echo "[IP]"
|
|
42: crowdsec:
|
|
67: docker compose up -d --force-recreate crowdsec
|
|
75: state="$(docker inspect crowdsec --format 'Running={{.State.Running}} Restarting={{.State.Restarting}} RestartCount={{.RestartCount}} StartedAt={{.State.StartedAt}} ExitCode={{.State.ExitCode}}' 2>/dev/null || true)"
|
|
76: health="$(curl -sk --connect-timeout 3 --max-time 6 -o /tmp/771m_health.html -w '%{http_code}' http://[IP]:8088/health || true)"
|
|
79: fatal="$(docker logs --since 30s crowdsec 2>&1 | grep -Eic 'fatal|unable to run local API|failed to get docker info|lookup socket-proxy|server misbehaving|mapping value is not allowed' || true)"
|
|
80: echo "LAPI_TRY=$n STATE=$state HEALTH=$health LAPI_RC=${lapi_rc:-NA} FATAL_COUNT_30S=$fatal"
|
|
84: && test "$health" = "200" \
|
|
127:remove("/opt/stacks/crowdsec/config/config.yaml")
|
|
128:remove("/opt/stacks/crowdsec/config/user.yaml")
|
|
135:p = Path("/opt/stacks/crowdsec/config/config.yaml")
|
|
166: " credentials_path: /etc/crowdsec/online_api_credentials.yaml",
|
|
178: crowdsec:
|
|
183: NO_PROXY: "[IP],localhost,[IP],[IPV6],socket-proxy,crowdsec"
|
|
245:collect_netbird_peers() {
|
|
246: echo "NETBIRD_STATUS"
|
|
247: if ! command -v netbird >/dev/null 2>&1; then
|
|
248: echo "NETBIRD_CLI_MISSING"
|
|
252: netbird status 2>&1 | redact | tee /tmp/771m_netbird_status.txt || true
|
|
253: netbird status --json >/tmp/771m_netbird_status.json 2>/tmp/771m_netbird_status_json.err || true
|
|
254: netbird ip 2>/tmp/771m_netbird_ip.err | tee /tmp/771m_netbird_ip.txt || true
|
|
256: echo "NETBIRD_INTERFACES"
|
|
257: ip -o -4 addr show | grep -Ei 'wt|netbird|wg|100\.' || true
|
|
259: python3 - <<'PY' >/tmp/771m_netbird_peers.tsv
|
|
264:p = Path("/tmp/771m_netbird_ip.txt")
|
|
278:jp = Path("/tmp/771m_netbird_status.json")
|
|
DEPENDENCY_FILE|path=/root/771n_netbird_usa_moldova_egress_only.sh|mtime=2026-07-02 18:21:39.746368388 +0300|sha256=420d835318960ae2644dafc93e0427584505acaffa9786927623aeb59ba6ac0e
|
|
8:PROOF="/root/evidence/771N_NETBIRD_USA_MOLDOVA_EGRESS_ONLY_${TS}_PROOF.txt"
|
|
10:REMOTE_SCRIPT="/tmp/771n_netbird_usa_moldova_egress_${TS}.sh"
|
|
11:LOCAL_REMOTE_SCRIPT="/root/771n_netbird_usa_moldova_egress_remote_${TS}.sh"
|
|
18:STACK=/opt/stacks/crowdsec
|
|
20:EGRESS_DIR=/etc/homelab-crowdsec-capi-netbird-egress
|
|
21:HEALTH_DIR=/var/lib/homelab-health
|
|
22:HEALTH_FILE="$HEALTH_DIR/crowdsec-capi.txt"
|
|
25:mkdir -p "$EGRESS_DIR/profiles" "$HEALTH_DIR"
|
|
32: docker network inspect crowdsec_default --format '{{(index .IPAM.Config 0).Gateway}}' 2>/dev/null || echo "[IP]"
|
|
35:write_health() {
|
|
41: echo "CHECK=crowdsec-capi"
|
|
44: } > "$HEALTH_FILE"
|
|
47:force_crowdsec_safe() {
|
|
51: crowdsec:
|
|
74: docker compose up -d crowdsec
|
|
84: cat >"$EGRESS_DIR/profiles/${profile}.env" <<EOF
|
|
91: cp -a "$EGRESS_DIR/profiles/${profile}.env" "$EGRESS_DIR/current.env"
|
|
93: cat >/etc/systemd/system/homelab-crowdsec-capi-netbird-egress.service <<'EOF'
|
|
95:Description=Homelab CrowdSec CAPI NetBird VPS SOCKS egress
|
|
96:After=network-online.target docker.service netbird.service
|
|
102:EnvironmentFile=/etc/homelab-crowdsec-capi-netbird-egress/current.env
|
|
111: cat >/usr/local/sbin/homelab-crowdsec-capi-egress-switch <<'EOF'
|
|
115:base=/etc/homelab-crowdsec-capi-netbird-egress
|
|
123:systemctl restart homelab-crowdsec-capi-netbird-egress.service
|
|
126:systemctl --no-pager --full status homelab-crowdsec-capi-netbird-egress.service | sed -n '1,30p'
|
|
128: chmod 755 /usr/local/sbin/homelab-crowdsec-capi-egress-switch
|
|
131: systemctl enable --now homelab-crowdsec-capi-netbird-egress.service
|
|
132: systemctl restart homelab-crowdsec-capi-netbird-egress.service
|
|
134: systemctl is-active homelab-crowdsec-capi-netbird-egress.service || true
|
|
146: if "HOMELAB_771" in line and "CROWDSEC_CAPI" in line and "BEGIN" in line:
|
|
149: if "HOMELAB_771" in line and "CROWDSEC_CAPI" in line and "END" in line:
|
|
160:out.append("# HOMELAB_771N_CROWDSEC_CAPI_NETBIRD_BEGIN")
|
|
163:out.append("# HOMELAB_771N_CROWDSEC_CAPI_NETBIRD_END")
|
|
194: code="$(curl -sk --proxy "socks5h://[IP]:${port}" --connect-timeout 12 --max-time 45 -o /tmp/771n_capi_${profile}.body -w '%{http_code}' https://api.crowdsec.net/v3/watchers/login || true)"
|
|
203: echo "VPS_EGRESS_OK profile=$profile user=$user ip=$ip"
|
|
214: echo "VPS_EGRESS_NOT_READY profile=$profile ip=$ip"
|
|
DEPENDENCY_FILE|path=/root/771n_netbird_usa_moldova_egress_remote_20260702T152139Z.sh|mtime=2026-07-02 18:21:39.750535800 +0300|sha256=1c25cbc13f524d0f6974a71c255c5634c69380818cb1b5ad4f8f6ea9bf8c6a01
|
|
5:STACK=/opt/stacks/crowdsec
|
|
7:EGRESS_DIR=/etc/homelab-crowdsec-capi-netbird-egress
|
|
8:HEALTH_DIR=/var/lib/homelab-health
|
|
9:HEALTH_FILE="$HEALTH_DIR/crowdsec-capi.txt"
|
|
12:mkdir -p "$EGRESS_DIR/profiles" "$HEALTH_DIR"
|
|
19: docker network inspect crowdsec_default --format '{{(index .IPAM.Config 0).Gateway}}' 2>/dev/null || echo "[IP]"
|
|
22:write_health() {
|
|
28: echo "CHECK=crowdsec-capi"
|
|
31: } > "$HEALTH_FILE"
|
|
34:force_crowdsec_safe() {
|
|
38: crowdsec:
|
|
61: docker compose up -d crowdsec
|
|
71: cat >"$EGRESS_DIR/profiles/${profile}.env" <<EOF
|
|
78: cp -a "$EGRESS_DIR/profiles/${profile}.env" "$EGRESS_DIR/current.env"
|
|
80: cat >/etc/systemd/system/homelab-crowdsec-capi-netbird-egress.service <<'EOF'
|
|
82:Description=Homelab CrowdSec CAPI NetBird VPS SOCKS egress
|
|
83:After=network-online.target docker.service netbird.service
|
|
89:EnvironmentFile=/etc/homelab-crowdsec-capi-netbird-egress/current.env
|
|
98: cat >/usr/local/sbin/homelab-crowdsec-capi-egress-switch <<'EOF'
|
|
102:base=/etc/homelab-crowdsec-capi-netbird-egress
|
|
110:systemctl restart homelab-crowdsec-capi-netbird-egress.service
|
|
113:systemctl --no-pager --full status homelab-crowdsec-capi-netbird-egress.service | sed -n '1,30p'
|
|
115: chmod 755 /usr/local/sbin/homelab-crowdsec-capi-egress-switch
|
|
118: systemctl enable --now homelab-crowdsec-capi-netbird-egress.service
|
|
119: systemctl restart homelab-crowdsec-capi-netbird-egress.service
|
|
121: systemctl is-active homelab-crowdsec-capi-netbird-egress.service || true
|
|
133: if "HOMELAB_771" in line and "CROWDSEC_CAPI" in line and "BEGIN" in line:
|
|
136: if "HOMELAB_771" in line and "CROWDSEC_CAPI" in line and "END" in line:
|
|
147:out.append("# HOMELAB_771N_CROWDSEC_CAPI_NETBIRD_BEGIN")
|
|
150:out.append("# HOMELAB_771N_CROWDSEC_CAPI_NETBIRD_END")
|
|
181: code="$(curl -sk --proxy "socks5h://[IP]:${port}" --connect-timeout 12 --max-time 45 -o /tmp/771n_capi_${profile}.body -w '%{http_code}' https://api.crowdsec.net/v3/watchers/login || true)"
|
|
190: echo "VPS_EGRESS_OK profile=$profile user=$user ip=$ip"
|
|
201: echo "VPS_EGRESS_NOT_READY profile=$profile ip=$ip"
|
|
206:echo "STEP=771N_NETBIRD_USA_MOLDOVA_EGRESS_ONLY"
|
|
207:echo "RULE=WARP_PURGED_NETBIRD_ONLY"
|
|
215:echo "CROWDSEC_SAFE_STATE"
|
|
DEPENDENCY_FILE|path=/root/771o_netbird_management_acl_discovery.sh|mtime=2026-07-02 18:24:39.066582039 +0300|sha256=50526afeaeb99fa9a8018374abe82731dcb9e93eac4be64934d0c6a8610be322
|
|
8:PROOF="/root/evidence/771O_NETBIRD_MANAGEMENT_ACL_DISCOVERY_${TS}_PROOF.txt"
|
|
14: echo "STEP=771O_NETBIRD_MANAGEMENT_ACL_DISCOVERY"
|
|
17: echo "RULE=WARP_PURGED_NETBIRD_POLICY_DISCOVERY_ONLY"
|
|
24: echo "EDGE_NETBIRD_ACL_DIAG"
|
|
31: echo NETBIRD_STATUS
|
|
34: echo NETBIRD_JSON_TARGETS
|
|
35: netbird status --json >/tmp/771o_netbird_status.json 2>/tmp/771o_netbird_status_json.err || true
|
|
39:p=Path(\"/tmp/771o_netbird_status.json\")
|
|
42: print(Path(\"/tmp/771o_netbird_status_json.err\").read_text(errors=\"ignore\")[:500] if Path(\"/tmp/771o_netbird_status_json.err\").exists() else \"\")
|
|
46:names=(\"e3qxxx\",\"183-106\",\"moldova\",\"usa\")
|
|
82: echo CROWDSEC_SAFE_STATE
|
|
83: sudo docker ps --format \"{{.Names}} {{.Status}} {{.Ports}}\" | grep -E \"^(crowdsec|socket-proxy|npmplus)\" || true
|
|
87: echo "DISCOVER_NETBIRD_MANAGEMENT_ON_KNOWN_HOSTS"
|
|
101: echo DOCKER_NETBIRD_CONTAINERS
|
|
102: docker ps -a --format "{{.Names}} {{.Image}} {{.Status}} {{.Ports}}" 2>/dev/null | grep -Ei "netbird|management|signal|relay|coturn|zitadel|dashboard" || true
|
|
103: echo SYSTEMD_NETBIRD
|
|
104: systemctl list-units --all --no-pager 2>/dev/null | grep -Ei "netbird|signal|relay|coturn|zitadel" || true
|
|
106: find /opt /etc /var/lib -maxdepth 5 \( -iname "*netbird*" -o -iname "*management*.json" -o -iname "store.json" \) 2>/dev/null | sed -n "1,120p"
|
|
111: echo DOCKER_NETBIRD_CONTAINERS
|
|
112: docker ps -a --format \"{{.Names}} {{.Image}} {{.Status}} {{.Ports}}\" 2>/dev/null | grep -Ei \"netbird|management|signal|relay|coturn|zitadel|dashboard\" || true
|
|
113: echo SYSTEMD_NETBIRD
|
|
114: systemctl list-units --all --no-pager 2>/dev/null | grep -Ei \"netbird|signal|relay|coturn|zitadel\" || true
|
|
116: find /opt /etc /var/lib -maxdepth 5 \( -iname \"*netbird*\" -o -iname \"*management*.json\" -o -iname \"store.json\" \) 2>/dev/null | sed -n \"1,120p\"
|
|
122: echo "TRY_PARSE_NETBIRD_STORE_IF_FOUND_LOCALLY"
|
|
124: candidates=$(find /opt /etc /var/lib -maxdepth 7 \( -path "*netbird*" -a \( -iname "store.json" -o -iname "*.db" -o -iname "*.json" \) \) 2>/dev/null | sed -n "1,40p")
|
|
163: echo "STATUS=OK_771O_DISCOVERY_DONE_NEXT_FIX_NETBIRD_POLICY_OR_VPS_AGENT"
|
|
DEPENDENCY_FILE|path=/root/771p_netbird_vps_peer_repair.sh|mtime=2026-07-02 19:32:10.024856543 +0300|sha256=f37aac25cac5b1c3983392070ddaff09e28497efa28884b03d23efc09a825747
|
|
8:PROOF="/root/evidence/771P_NETBIRD_VPS_PEER_REPAIR_${TS}_PROOF.txt"
|
|
11:MOLDOVA_NB=[IP]
|
|
16: echo "STEP=771P_NETBIRD_VPS_PEER_REPAIR"
|
|
19: echo "RULE=NO_WARP_FIX_NETBIRD_USA_MOLDOVA_PEERS_FOR_CROWDSEC_CAPI"
|
|
23: echo "CONFIG_CHANGE=YES_RESTART_NETBIRD_ON_VPS_IF_PUBLIC_SSH_FOUND"
|
|
26: echo "LOCAL_PVE01_NETBIRD_STATUS"
|
|
29: for ip in "$USA_NB" "$MOLDOVA_NB"; do
|
|
35: echo "EDGE_NETBIRD_STATUS_AND_TARGETS"
|
|
42: echo NETBIRD_STATUS
|
|
46: netbird status --json >/tmp/771p_edge_nb.json 2>/tmp/771p_edge_nb.err || true
|
|
57: ip=peer.get(\"netbirdIp\",\"\")
|
|
59: print(\"PEER\", fqdn, ip, \"status=\"+str(peer.get(\"status\")), \"handshake=\"+str(peer.get(\"lastWireguardHandshake\")), \"relay=\"+str(peer.get(\"relayAddress\")), \"conn=\"+str(peer.get(\"connectionType\")))
|
|
80: grep -nEi 'e3qxxx|183-106|moldova|молдов|usa|america|vps|netbird|alexhost|aeza|hetzner|public ip|external ip|ssh' /etc/pve/31_HOMELAB_REFERENCE.md 2>/dev/null | sed -n '1,260p' || true
|
|
81: echo "SOURCE=systemd_netbird_vps"
|
|
84: grep -RHEi 'Host |HostName|User |Port |e3qxxx|183-106|moldova|usa|vps' [SENSITIVE_PATH] /etc/ssh/ssh_config /etc/ssh/ssh_config.d/* 2>/dev/null | sed -n '1,260p' || true
|
|
110: if any(x in low for x in ["183-106", "moldova", "молдов", "chisinau"]):
|
|
111: profile = "moldova"
|
|
123: if "netbird.selfhosted" in host:
|
|
138: echo "TEST_PUBLIC_SSH_AND_RESTART_NETBIRD_ON_VPS_IF_FOUND"
|
|
159: systemctl restart netbird 2>/dev/null || true
|
|
173: echo "CROWDSEC_SAFE_POSTCHECK"
|
|
175: sudo docker ps --format \"{{.Names}} {{.Status}} {{.Ports}}\" | grep -E \"^(crowdsec|socket-proxy|npmplus)\" || true
|
|
182: echo "STATUS=OK_771P_VPS_PUBLIC_ACCESS_FOUND_RESTARTED_NETBIRD_REVIEW_RECHECK"
|
|
DEPENDENCY_FILE|path=/root/771q_netbird_public_vps_repair_then_capi.sh|mtime=2026-07-02 19:45:41.256069566 +0300|sha256=0b6e42e5b4013f7ee2191b140c5c507877fb05e2a4fdd7b39bea7b7c6b108071
|
|
8:PROOF="/root/evidence/771Q_NETBIRD_PUBLIC_VPS_REPAIR_THEN_CAPI_${TS}_PROOF.txt"
|
|
11:MOLDOVA_NB=[IP]
|
|
16: echo "STEP=771Q_NETBIRD_PUBLIC_VPS_REPAIR_THEN_CAPI"
|
|
19: echo "RULE=NO_WARP_REPAIR_NETBIRD_USA_MOLDOVA_THEN_CROWDSEC_CAPI"
|
|
23: echo "CONFIG_CHANGE=YES_RESTART_PUBLIC_VPS_NETBIRD_AND_CAPI_IF_ROUTE_READY"
|
|
38: "echo SSH_OK; echo HOSTNAME=\$(hostname); command -v netbird >/dev/null 2>&1 && netbird status 2>&1 || echo NETBIRD_CLI_MISSING; curl -4 -sk --connect-timeout 8 --max-time 20 -o /tmp/771q_capi.body -w 'DIRECT_CAPI_HTTP=%{http_code}\n' https://api.crowdsec.net/v3/watchers/login || true" \
|
|
58: "moldova|debian|netbird.vpn.gram1.ru" \
|
|
59: "moldova|root|netbird.vpn.gram1.ru" \
|
|
60: "moldova|debian|[IP]" \
|
|
61: "moldova|root|[IP]" \
|
|
62: "netbird-ui|debian|nb.pvepro.ru" \
|
|
63: "netbird-ui|root|nb.pvepro.ru"
|
|
74: echo "RESTART_NETBIRD_ON_REAL_PUBLIC_VPS_ONLY"
|
|
79: netbird.vpn.gram1.ru|[IP]|nb.pvepro.ru)
|
|
87: "echo BEFORE_HOSTNAME=\$(hostname); command -v netbird >/dev/null 2>&1 && netbird status 2>&1 || true; (sudo systemctl restart netbird 2>/dev/null || systemctl restart netbird 2>/dev/null || true); sleep 15; echo AFTER; command -v netbird >/dev/null 2>&1 && netbird status 2>&1 || true; curl -4 -sk --connect-timeout 8 --max-time 20 -o /tmp/771q_capi_after.body -w 'DIRECT_CAPI_HTTP_AFTER=%{http_code}\n' https://api.crowdsec.net/v3/watchers/login || true" \
|
|
106: echo "RECHECK_EDGE_USA_MOLDOVA_AFTER_PUBLIC_RESTART"
|
|
109: systemctl restart netbird 2>/dev/null || true
|
|
112: netbird status --json >/tmp/771q_edge_nb.json 2>/tmp/771q_edge_nb.err || true
|
|
121: ip=peer.get(\"netbirdIp\",\"\")
|
|
123: print(\"PEER\", fqdn, ip, \"status=\"+str(peer.get(\"status\")), \"handshake=\"+str(peer.get(\"lastWireguardHandshake\")), \"relay=\"+str(peer.get(\"relayAddress\")), \"conn=\"+str(peer.get(\"connectionType\")))
|
|
139: echo "EDGE_BUILD_NETBIRD_EGRESS_AND_REGISTER_CAPI_IF_REACHABLE"
|
|
144:STACK=/opt/stacks/crowdsec
|
|
146:EGRESS_DIR=/etc/homelab-crowdsec-capi-netbird-egress
|
|
147:HEALTH_DIR=/var/lib/homelab-health
|
|
148:HEALTH_FILE="$HEALTH_DIR/crowdsec-capi.txt"
|
|
151:mkdir -p "$EGRESS_DIR/profiles" "$HEALTH_DIR"
|
|
158: docker network inspect crowdsec_default --format '{{(index .IPAM.Config 0).Gateway}}' 2>/dev/null || echo "[IP]"
|
|
161:write_health() {
|
|
167: echo "CHECK=crowdsec-capi"
|
|
170: } > "$HEALTH_FILE"
|
|
177: crowdsec:
|
|
198: docker compose up -d crowdsec
|
|
206: health="$(curl -sk --connect-timeout 3 --max-time 6 -o /tmp/771q_health.html -w '%{http_code}' http://[IP]:8088/health || true)"
|
|
209: echo "LAPI_TRY=$n HEALTH=$health LAPI_RC=${rc:-NA}"
|
|
211: test "$health" = "200" && test "${rc:-NA}" = "0" && return 0
|
|
241:remove("/opt/stacks/crowdsec/config/config.yaml")
|
|
DEPENDENCY_FILE|path=/root/771r_edge_netbird_egress_capi_20260702T165517Z.sh|mtime=2026-07-02 19:55:17.909628261 +0300|sha256=9c6f911768869c984ec41016b3134be75620100fac8c60a8b21ef6bde58c6868
|
|
5:STACK=/opt/stacks/crowdsec
|
|
7:EGRESS_DIR=/etc/homelab-crowdsec-capi-netbird-egress
|
|
8:HEALTH_DIR=/var/lib/homelab-health
|
|
9:HEALTH_FILE="$HEALTH_DIR/crowdsec-capi.txt"
|
|
12:mkdir -p "$EGRESS_DIR/profiles" "$HEALTH_DIR"
|
|
19: docker network inspect crowdsec_default --format '{{(index .IPAM.Config 0).Gateway}}' 2>/dev/null || echo "[IP]"
|
|
22:write_health() {
|
|
29: echo "CHECK=crowdsec-capi"
|
|
32: } > "$HEALTH_FILE"
|
|
39: crowdsec:
|
|
62: docker compose up -d crowdsec
|
|
66: local label n health out rc fatal
|
|
71: health="$(curl -sk --connect-timeout 3 --max-time 6 -o /tmp/771r_health.html -w '%{http_code}' http://[IP]:8088/health || true)"
|
|
74: fatal="$(docker logs --since 30s crowdsec 2>&1 | grep -Eic 'fatal|unable to run local API|failed to get docker info|lookup socket-proxy|server misbehaving|mapping value is not allowed' || true)"
|
|
75: echo "LAPI_TRY=$n HEALTH=$health LAPI_RC=${rc:-NA} FATAL_COUNT_30S=$fatal"
|
|
77: if test "$health" = "200" && test "${rc:-NA}" = "0" && test "$fatal" = "0"; then
|
|
114:remove("/opt/stacks/crowdsec/config/config.yaml")
|
|
115:remove("/opt/stacks/crowdsec/config/user.yaml")
|
|
122:p = Path("/opt/stacks/crowdsec/config/config.yaml")
|
|
153: " credentials_path: /etc/crowdsec/online_api_credentials.yaml",
|
|
166: crowdsec:
|
|
171: NO_PROXY: "[IP],localhost,[IP],[IPV6],socket-proxy,crowdsec"
|
|
196:test_target_ssh_egress() {
|
|
202: echo "TEST_TARGET_SSH_EGRESS profile=$profile ip=$ip"
|
|
212: "echo SSH_OK; hostname; curl -4 -sk --connect-timeout 8 --max-time 20 -o /tmp/771r_capi.body -w 'DIRECT_CAPI_HTTP=%{http_code}\n' https://api.crowdsec.net/v3/watchers/login || true" \
|
|
234: code="$(curl -sk --proxy "socks5h://[IP]:${port}" --connect-timeout 12 --max-time 45 -o /tmp/771r_capi_proxy.body -w '%{http_code}' https://api.crowdsec.net/v3/watchers/login || true)"
|
|
255:install_egress_service() {
|
|
261: echo "INSTALL_EGRESS profile=$profile user=$user ip=$ip bridge=$b"
|
|
263: mkdir -p "$EGRESS_DIR/profiles"
|
|
264: cat >"$EGRESS_DIR/profiles/${profile}.env" <<EOF
|
|
271: cp -a "$EGRESS_DIR/profiles/${profile}.env" "$EGRESS_DIR/current.env"
|
|
273: cat >/etc/systemd/system/homelab-crowdsec-capi-netbird-egress.service <<'EOF'
|
|
275:Description=Homelab CrowdSec CAPI NetBird VPS SOCKS egress
|
|
276:After=network-online.target docker.service netbird.service
|
|
282:EnvironmentFile=/etc/homelab-crowdsec-capi-netbird-egress/current.env
|
|
291: cat >/usr/local/sbin/homelab-crowdsec-capi-egress-switch <<'EOF'
|
|
DEPENDENCY_FILE|path=/root/771r_netbird_public_vps_fixed_then_capi.sh|mtime=2026-07-02 19:55:17.904566554 +0300|sha256=dc8119b6815d60e08f442e77faba6ce6e9899c4120d78490886e222b8510efa7
|
|
8:PROOF="/root/evidence/771R_NETBIRD_PUBLIC_VPS_FIXED_THEN_CAPI_${TS}_PROOF.txt"
|
|
10:EDGE_REMOTE="/tmp/771r_edge_netbird_egress_capi_${TS}.sh"
|
|
11:EDGE_LOCAL="/root/771r_edge_netbird_egress_capi_${TS}.sh"
|
|
31: "echo SSH_OK; echo USER=\$(id -un); echo HOSTNAME=\$(hostname); command -v netbird >/dev/null 2>&1 && netbird status 2>&1 || echo NETBIRD_CLI_MISSING; curl -4 -sk --connect-timeout 8 --max-time 20 -o /tmp/771r_capi.body -w 'DIRECT_CAPI_HTTP=%{http_code}\n' https://api.crowdsec.net/v3/watchers/login || true" \
|
|
49:STACK=/opt/stacks/crowdsec
|
|
51:EGRESS_DIR=/etc/homelab-crowdsec-capi-netbird-egress
|
|
52:HEALTH_DIR=/var/lib/homelab-health
|
|
53:HEALTH_FILE="$HEALTH_DIR/crowdsec-capi.txt"
|
|
56:mkdir -p "$EGRESS_DIR/profiles" "$HEALTH_DIR"
|
|
63: docker network inspect crowdsec_default --format '{{(index .IPAM.Config 0).Gateway}}' 2>/dev/null || echo "[IP]"
|
|
66:write_health() {
|
|
73: echo "CHECK=crowdsec-capi"
|
|
76: } > "$HEALTH_FILE"
|
|
83: crowdsec:
|
|
106: docker compose up -d crowdsec
|
|
110: local label n health out rc fatal
|
|
115: health="$(curl -sk --connect-timeout 3 --max-time 6 -o /tmp/771r_health.html -w '%{http_code}' http://[IP]:8088/health || true)"
|
|
118: fatal="$(docker logs --since 30s crowdsec 2>&1 | grep -Eic 'fatal|unable to run local API|failed to get docker info|lookup socket-proxy|server misbehaving|mapping value is not allowed' || true)"
|
|
119: echo "LAPI_TRY=$n HEALTH=$health LAPI_RC=${rc:-NA} FATAL_COUNT_30S=$fatal"
|
|
121: if test "$health" = "200" && test "${rc:-NA}" = "0" && test "$fatal" = "0"; then
|
|
158:remove("/opt/stacks/crowdsec/config/config.yaml")
|
|
159:remove("/opt/stacks/crowdsec/config/user.yaml")
|
|
166:p = Path("/opt/stacks/crowdsec/config/config.yaml")
|
|
197: " credentials_path: /etc/crowdsec/online_api_credentials.yaml",
|
|
210: crowdsec:
|
|
215: NO_PROXY: "[IP],localhost,[IP],[IPV6],socket-proxy,crowdsec"
|
|
240:test_target_ssh_egress() {
|
|
246: echo "TEST_TARGET_SSH_EGRESS profile=$profile ip=$ip"
|
|
256: "echo SSH_OK; hostname; curl -4 -sk --connect-timeout 8 --max-time 20 -o /tmp/771r_capi.body -w 'DIRECT_CAPI_HTTP=%{http_code}\n' https://api.crowdsec.net/v3/watchers/login || true" \
|
|
278: code="$(curl -sk --proxy "socks5h://[IP]:${port}" --connect-timeout 12 --max-time 45 -o /tmp/771r_capi_proxy.body -w '%{http_code}' https://api.crowdsec.net/v3/watchers/login || true)"
|
|
299:install_egress_service() {
|
|
305: echo "INSTALL_EGRESS profile=$profile user=$user ip=$ip bridge=$b"
|
|
307: mkdir -p "$EGRESS_DIR/profiles"
|
|
308: cat >"$EGRESS_DIR/profiles/${profile}.env" <<EOF
|
|
315: cp -a "$EGRESS_DIR/profiles/${profile}.env" "$EGRESS_DIR/current.env"
|
|
317: cat >/etc/systemd/system/homelab-crowdsec-capi-netbird-egress.service <<'EOF'
|
|
DEPENDENCY_FILE|path=/root/771s_edge_capi_netbird_20260702T170251Z.sh|mtime=2026-07-02 20:02:51.044349876 +0300|sha256=c4783c14a3a132616af5db6a065270ccd39a7690f9fd38d0ef27fe5de6bd07e5
|
|
6:PROFILE="${3:-moldova}"
|
|
7:STACK=/opt/stacks/crowdsec
|
|
9:EGRESS_DIR=/etc/homelab-crowdsec-capi-netbird-egress
|
|
10:HEALTH_DIR=/var/lib/homelab-health
|
|
11:HEALTH_FILE="$HEALTH_DIR/crowdsec-capi.txt"
|
|
14:mkdir -p "$EGRESS_DIR/profiles" "$HEALTH_DIR"
|
|
21: docker network inspect crowdsec_default --format '{{(index .IPAM.Config 0).Gateway}}' 2>/dev/null || echo "[IP]"
|
|
24:write_health() {
|
|
30: echo "CHECK=crowdsec-capi"
|
|
33: } > "$HEALTH_FILE"
|
|
40: crowdsec:
|
|
62: docker compose up -d crowdsec
|
|
70: health="$(curl -sk --connect-timeout 3 --max-time 6 -o /tmp/771s_health.html -w '%{http_code}' http://[IP]:8088/health || true)"
|
|
73: fatal="$(docker logs --since 30s crowdsec 2>&1 | grep -Eic 'fatal|unable to run local API|failed to get docker info|lookup socket-proxy|server misbehaving|mapping value is not allowed' || true)"
|
|
74: echo "LAPI_TRY=$n HEALTH=$health LAPI_RC=${rc:-NA} FATAL_COUNT_30S=$fatal"
|
|
76: if test "$health" = "200" && test "${rc:-NA}" = "0" && test "$fatal" = "0"; then
|
|
113:remove("/opt/stacks/crowdsec/config/config.yaml")
|
|
114:remove("/opt/stacks/crowdsec/config/user.yaml")
|
|
121:p=Path("/opt/stacks/crowdsec/config/config.yaml")
|
|
153: " credentials_path: /etc/crowdsec/online_api_credentials.yaml",
|
|
165: crowdsec:
|
|
170: NO_PROXY: "[IP],localhost,[IP],[IPV6],socket-proxy,crowdsec"
|
|
195:test_target_ssh_egress() {
|
|
199: echo "TEST_TARGET_SSH_EGRESS profile=$profile ip=$ip"
|
|
209: "echo SSH_OK; hostname; curl -4 -sk --connect-timeout 8 --max-time 20 -o /tmp/771s_capi.body -w 'DIRECT_CAPI_HTTP=%{http_code}\n' https://api.crowdsec.net/v3/watchers/login || true" \
|
|
231: code="$(curl -sk --proxy "socks5h://[IP]:${port}" --connect-timeout 12 --max-time 45 -o /tmp/771s_capi_proxy.body -w '%{http_code}' https://api.crowdsec.net/v3/watchers/login || true)"
|
|
252:install_egress_service() {
|
|
257: echo "INSTALL_EGRESS profile=$profile user=$user ip=$ip bridge=$b"
|
|
259: mkdir -p "$EGRESS_DIR/profiles"
|
|
260: cat >"$EGRESS_DIR/profiles/${profile}.env" <<EOF
|
|
267: cp -a "$EGRESS_DIR/profiles/${profile}.env" "$EGRESS_DIR/current.env"
|
|
269: cat >/etc/systemd/system/homelab-crowdsec-capi-netbird-egress.service <<'EOF'
|
|
271:Description=Homelab CrowdSec CAPI NetBird VPS SOCKS egress
|
|
272:After=network-online.target docker.service netbird.service
|
|
278:EnvironmentFile=/etc/homelab-crowdsec-capi-netbird-egress/current.env
|
|
287: cat >/usr/local/sbin/homelab-crowdsec-capi-egress-switch <<'EOF'
|
|
DEPENDENCY_FILE|path=/root/771s_vps_identity_repair_netbird_capi.sh|mtime=2026-07-02 20:02:51.038348586 +0300|sha256=16121a810320b1517291830ba128baf6c68e7b3e6f78786481dedb78a1f061b9
|
|
8:PROOF="/root/evidence/771S_VPS_IDENTITY_REPAIR_NETBIRD_CAPI_${TS}_PROOF.txt"
|
|
11:VPS_ALIAS=netbird.vpn.gram1.ru
|
|
13:EDGE_REMOTE="/tmp/771s_edge_capi_netbird_${TS}.sh"
|
|
14:EDGE_LOCAL="/root/771s_edge_capi_netbird_${TS}.sh"
|
|
28:PROFILE="${3:-moldova}"
|
|
29:STACK=/opt/stacks/crowdsec
|
|
31:EGRESS_DIR=/etc/homelab-crowdsec-capi-netbird-egress
|
|
32:HEALTH_DIR=/var/lib/homelab-health
|
|
33:HEALTH_FILE="$HEALTH_DIR/crowdsec-capi.txt"
|
|
36:mkdir -p "$EGRESS_DIR/profiles" "$HEALTH_DIR"
|
|
43: docker network inspect crowdsec_default --format '{{(index .IPAM.Config 0).Gateway}}' 2>/dev/null || echo "[IP]"
|
|
46:write_health() {
|
|
52: echo "CHECK=crowdsec-capi"
|
|
55: } > "$HEALTH_FILE"
|
|
62: crowdsec:
|
|
84: docker compose up -d crowdsec
|
|
92: health="$(curl -sk --connect-timeout 3 --max-time 6 -o /tmp/771s_health.html -w '%{http_code}' http://[IP]:8088/health || true)"
|
|
95: fatal="$(docker logs --since 30s crowdsec 2>&1 | grep -Eic 'fatal|unable to run local API|failed to get docker info|lookup socket-proxy|server misbehaving|mapping value is not allowed' || true)"
|
|
96: echo "LAPI_TRY=$n HEALTH=$health LAPI_RC=${rc:-NA} FATAL_COUNT_30S=$fatal"
|
|
98: if test "$health" = "200" && test "${rc:-NA}" = "0" && test "$fatal" = "0"; then
|
|
135:remove("/opt/stacks/crowdsec/config/config.yaml")
|
|
136:remove("/opt/stacks/crowdsec/config/user.yaml")
|
|
143:p=Path("/opt/stacks/crowdsec/config/config.yaml")
|
|
175: " credentials_path: /etc/crowdsec/online_api_credentials.yaml",
|
|
187: crowdsec:
|
|
192: NO_PROXY: "[IP],localhost,[IP],[IPV6],socket-proxy,crowdsec"
|
|
217:test_target_ssh_egress() {
|
|
221: echo "TEST_TARGET_SSH_EGRESS profile=$profile ip=$ip"
|
|
231: "echo SSH_OK; hostname; curl -4 -sk --connect-timeout 8 --max-time 20 -o /tmp/771s_capi.body -w 'DIRECT_CAPI_HTTP=%{http_code}\n' https://api.crowdsec.net/v3/watchers/login || true" \
|
|
253: code="$(curl -sk --proxy "socks5h://[IP]:${port}" --connect-timeout 12 --max-time 45 -o /tmp/771s_capi_proxy.body -w '%{http_code}' https://api.crowdsec.net/v3/watchers/login || true)"
|
|
274:install_egress_service() {
|
|
279: echo "INSTALL_EGRESS profile=$profile user=$user ip=$ip bridge=$b"
|
|
281: mkdir -p "$EGRESS_DIR/profiles"
|
|
282: cat >"$EGRESS_DIR/profiles/${profile}.env" <<EOF
|
|
289: cp -a "$EGRESS_DIR/profiles/${profile}.env" "$EGRESS_DIR/current.env"
|
|
291: cat >/etc/systemd/system/homelab-crowdsec-capi-netbird-egress.service <<'EOF'
|
|
DEPENDENCY_FILE|path=/root/771t_temp_hostkey_netbird_identity_then_capi.sh|mtime=2026-07-02 20:10:05.923400398 +0300|sha256=0c3d628b23b67849d52112322ab2e1ee3779012c10578a0b8cece0d8ddf639da
|
|
8:PROOF="/root/evidence/771T_TEMP_HOSTKEY_NETBIRD_IDENTITY_THEN_CAPI_${TS}_PROOF.txt"
|
|
29:STACK=/opt/stacks/crowdsec
|
|
31:EGRESS_DIR=/etc/homelab-crowdsec-capi-netbird-egress
|
|
32:HEALTH_DIR=/var/lib/homelab-health
|
|
33:HEALTH_FILE="$HEALTH_DIR/crowdsec-capi.txt"
|
|
36:mkdir -p "$EGRESS_DIR/profiles" "$HEALTH_DIR"
|
|
43: docker network inspect crowdsec_default --format '{{(index .IPAM.Config 0).Gateway}}' 2>/dev/null || echo "[IP]"
|
|
46:write_health() {
|
|
50: echo "CHECK=crowdsec-capi"
|
|
53: } > "$HEALTH_FILE"
|
|
60: crowdsec:
|
|
82: docker compose up -d crowdsec
|
|
90: health="$(curl -sk --connect-timeout 3 --max-time 6 -o /tmp/771t_health.html -w '%{http_code}' http://[IP]:8088/health || true)"
|
|
93: fatal="$(docker logs --since 30s crowdsec 2>&1 | grep -Eic 'fatal|unable to run local API|failed to get docker info|lookup socket-proxy|server misbehaving|mapping value is not allowed' || true)"
|
|
94: echo "LAPI_TRY=$n HEALTH=$health LAPI_RC=${rc:-NA} FATAL_COUNT_30S=$fatal"
|
|
96: if test "$health" = "200" && test "${rc:-NA}" = "0" && test "$fatal" = "0"; then
|
|
131:remove("/opt/stacks/crowdsec/config/config.yaml")
|
|
132:remove("/opt/stacks/crowdsec/config/user.yaml")
|
|
139:p=Path("/opt/stacks/crowdsec/config/config.yaml")
|
|
169: " credentials_path: /etc/crowdsec/online_api_credentials.yaml",
|
|
181: crowdsec:
|
|
186: NO_PROXY: "[IP],localhost,[IP],[IPV6],socket-proxy,crowdsec"
|
|
211:install_egress() {
|
|
213: echo "INSTALL_EGRESS target=$TARGET_NB user=$TARGET_USER profile=$PROFILE bridge=$b"
|
|
215: echo "VERIFY_NETBIRD_SSH_TO_TARGET"
|
|
218: "echo SSH_OK; hostname; curl -4 -sk --connect-timeout 8 --max-time 20 -o /tmp/771t_capi_direct.body -w 'DIRECT_CAPI_HTTP=%{http_code}\n' https://api.crowdsec.net/v3/watchers/login || true" >"$out" 2>&1
|
|
226: mkdir -p "$EGRESS_DIR/profiles"
|
|
227: cat >"$EGRESS_DIR/profiles/${PROFILE}.env" <<EOF
|
|
234: cp -a "$EGRESS_DIR/profiles/${PROFILE}.env" "$EGRESS_DIR/current.env"
|
|
236: cat >/etc/systemd/system/homelab-crowdsec-capi-netbird-egress.service <<'EOF'
|
|
238:Description=Homelab CrowdSec CAPI NetBird VPS SOCKS egress
|
|
239:After=network-online.target docker.service netbird.service
|
|
245:EnvironmentFile=/etc/homelab-crowdsec-capi-netbird-egress/current.env
|
|
254: cat >/usr/local/sbin/homelab-crowdsec-capi-egress-switch <<'EOF'
|
|
258:base=/etc/homelab-crowdsec-capi-netbird-egress
|
|
266:systemctl restart homelab-crowdsec-capi-netbird-egress.service
|
|
DEPENDENCY_FILE|path=/root/771u_find_key_fix_netbird_egress_capi.sh|mtime=2026-07-02 20:19:23.540526618 +0300|sha256=6b6511ec3d614793e3542777ccfcb5e8c5f09cff77ee2d57624b3ac980787bcb
|
|
8:PROOF="/root/evidence/771U_FIND_KEY_FIX_NETBIRD_EGRESS_CAPI_${TS}_PROOF.txt"
|
|
29:STACK=/opt/stacks/crowdsec
|
|
31:EGRESS_DIR=/etc/homelab-crowdsec-capi-netbird-egress
|
|
32:HEALTH_DIR=/var/lib/homelab-health
|
|
33:HEALTH_FILE="$HEALTH_DIR/crowdsec-capi.txt"
|
|
34:SSH_KEY="$EGRESS_DIR/id_ed25519"
|
|
37:mkdir -p "$EGRESS_DIR/profiles" "$HEALTH_DIR"
|
|
38:chmod 700 "$EGRESS_DIR"
|
|
45: docker network inspect crowdsec_default --format '{{(index .IPAM.Config 0).Gateway}}' 2>/dev/null || echo "[IP]"
|
|
48:write_health() {
|
|
52: echo "CHECK=crowdsec-capi"
|
|
55: } > "$HEALTH_FILE"
|
|
62: crowdsec:
|
|
84: docker compose up -d crowdsec
|
|
92: health="$(curl -sk --connect-timeout 3 --max-time 6 -o /tmp/771u_health.html -w '%{http_code}' http://[IP]:8088/health || true)"
|
|
95: fatal="$(docker logs --since 30s crowdsec 2>&1 | grep -Eic 'fatal|unable to run local API|failed to get docker info|lookup socket-proxy|server misbehaving|mapping value is not allowed' || true)"
|
|
96: echo "LAPI_TRY=$n HEALTH=$health LAPI_RC=${rc:-NA} FATAL_COUNT_30S=$fatal"
|
|
98: if test "$health" = "200" && test "${rc:-NA}" = "0" && test "$fatal" = "0"; then
|
|
133:remove("/opt/stacks/crowdsec/config/config.yaml")
|
|
134:remove("/opt/stacks/crowdsec/config/user.yaml")
|
|
141:p=Path("/opt/stacks/crowdsec/config/config.yaml")
|
|
170: " credentials_path: /etc/crowdsec/online_api_credentials.yaml",
|
|
182: crowdsec:
|
|
187: NO_PROXY: "[IP],localhost,[IP],[IPV6],socket-proxy,crowdsec"
|
|
213: ssh-keygen -q -t ed25519 -N "" -C "homelab-crowdsec-capi-netbird-egress-edge" -f "$SSH_KEY"
|
|
217: echo "EDGE_EGRESS_PUBKEY_FINGERPRINT"
|
|
221:install_egress() {
|
|
223: echo "INSTALL_EGRESS target=$TARGET_NB user=$TARGET_USER profile=$PROFILE bridge=$b"
|
|
225: echo "VERIFY_NETBIRD_SSH_TO_TARGET"
|
|
234: "echo SSH_OK; hostname; curl -4 -sk --connect-timeout 8 --max-time 20 -o /tmp/771u_capi_direct.body -w 'DIRECT_CAPI_HTTP=%{http_code}\n' https://api.crowdsec.net/v3/watchers/login || true" >"$out" 2>&1
|
|
242: mkdir -p "$EGRESS_DIR/profiles"
|
|
243: cat >"$EGRESS_DIR/profiles/${PROFILE}.env" <<EOF
|
|
251: cp -a "$EGRESS_DIR/profiles/${PROFILE}.env" "$EGRESS_DIR/current.env"
|
|
253: cat >/etc/systemd/system/homelab-crowdsec-capi-netbird-egress.service <<'EOF'
|
|
255:Description=Homelab CrowdSec CAPI NetBird VPS SOCKS egress
|
|
256:After=network-online.target docker.service netbird.service
|
|
DEPENDENCY_FILE|path=/root/771v_deep_key_backup_netbird_capi.sh|mtime=2026-07-02 20:31:17.990220570 +0300|sha256=92191ce6124981ee0468cc8f95c749c47bca5fc61f9aecaeedeef91cdd434170
|
|
8:PROOF="/root/evidence/771V_DEEP_KEY_BACKUP_NETBIRD_CAPI_${TS}_PROOF.txt"
|
|
11:EDGE_LOCAL="/root/771v_edge_netbird_capi_${TS}.sh"
|
|
12:EDGE_REMOTE="/tmp/771v_edge_netbird_capi_${TS}.sh"
|
|
29:STACK=/opt/stacks/crowdsec
|
|
31:EGRESS_DIR=/etc/homelab-crowdsec-capi-netbird-egress
|
|
32:HEALTH_DIR=/var/lib/homelab-health
|
|
33:HEALTH_FILE="$HEALTH_DIR/crowdsec-capi.txt"
|
|
34:SSH_KEY="$EGRESS_DIR/id_ed25519"
|
|
37:mkdir -p "$EGRESS_DIR/profiles" "$HEALTH_DIR"
|
|
38:chmod 700 "$EGRESS_DIR"
|
|
45: docker network inspect crowdsec_default --format '{{(index .IPAM.Config 0).Gateway}}' 2>/dev/null || echo "[IP]"
|
|
48:write_health() {
|
|
52: echo "CHECK=crowdsec-capi"
|
|
55: } > "$HEALTH_FILE"
|
|
62: crowdsec:
|
|
84: docker compose up -d crowdsec
|
|
92: health="$(curl -sk --connect-timeout 3 --max-time 6 -o /tmp/771v_health.html -w '%{http_code}' http://[IP]:8088/health || true)"
|
|
95: fatal="$(docker logs --since 30s crowdsec 2>&1 | grep -Eic 'fatal|unable to run local API|failed to get docker info|lookup socket-proxy|server misbehaving|mapping value is not allowed' || true)"
|
|
96: echo "LAPI_TRY=$n HEALTH=$health LAPI_RC=${rc:-NA} FATAL_COUNT_30S=$fatal"
|
|
98: if test "$health" = "200" && test "${rc:-NA}" = "0" && test "$fatal" = "0"; then
|
|
133:remove("/opt/stacks/crowdsec/config/config.yaml")
|
|
134:remove("/opt/stacks/crowdsec/config/user.yaml")
|
|
141:p = Path("/opt/stacks/crowdsec/config/config.yaml")
|
|
172: " credentials_path: /etc/crowdsec/online_api_credentials.yaml",
|
|
184: crowdsec:
|
|
189: NO_PROXY: "[IP],localhost,[IP],[IPV6],socket-proxy,crowdsec"
|
|
216: ssh-keygen -q -t ed25519 -N "" -C "homelab-crowdsec-capi-netbird-egress-edge" -f "$SSH_KEY"
|
|
220: echo "EDGE_EGRESS_PUBKEY_FINGERPRINT"
|
|
224:install_egress() {
|
|
226: echo "INSTALL_EGRESS target=$TARGET_NB user=$TARGET_USER profile=$PROFILE bridge=$b"
|
|
228: echo "VERIFY_NETBIRD_SSH_TO_TARGET"
|
|
237: "echo SSH_OK; hostname; curl -4 -sk --connect-timeout 8 --max-time 20 -o /tmp/771v_capi_direct.body -w 'DIRECT_CAPI_HTTP=%{http_code}\n' https://api.crowdsec.net/v3/watchers/login || true" >"$out" 2>&1
|
|
245: mkdir -p "$EGRESS_DIR/profiles"
|
|
246: cat >"$EGRESS_DIR/profiles/${PROFILE}.env" <<EOF
|
|
254: cp -a "$EGRESS_DIR/profiles/${PROFILE}.env" "$EGRESS_DIR/current.env"
|
|
256: cat >/etc/systemd/system/homelab-crowdsec-capi-netbird-egress.service <<'EOF'
|
|
DEPENDENCY_FILE|path=/root/771z_after_manual_netbird_egress_capi.sh|mtime=2026-07-02 21:37:47.943243883 +0300|sha256=f198ed621fd726ed6f15c4a0dd49fad307befd01e7a58ebbda1ddc379f5b49ee
|
|
8:PROOF="/root/evidence/771Z_AFTER_MANUAL_NETBIRD_EGRESS_CAPI_${TS}_PROOF.txt"
|
|
14: echo "STEP=771Z_AFTER_MANUAL_NETBIRD_EGRESS_CAPI"
|
|
17: echo "RULE=VERIFY_RELAY_MAIL_EGRESS_AND_REGISTER_CROWDSEC_CAPI"
|
|
18: echo "TARGETS=edge-vm:[IP] relay-moldova:[IP] mail-usa:[IP]"
|
|
25:STACK=/opt/stacks/crowdsec
|
|
27:HEALTH=/var/lib/homelab-health/crowdsec-capi.txt
|
|
28:mkdir -p /var/lib/homelab-health
|
|
34:write_health() {
|
|
38: echo "CHECK=crowdsec-capi"
|
|
41: } > "$HEALTH"
|
|
49: health="$(curl -sk --connect-timeout 3 --max-time 6 -o /tmp/771z_health.html -w '%{http_code}' http://[IP]:8088/health || true)"
|
|
52: echo "LAPI_TRY=$n HEALTH=$health LAPI_RC=${rc:-NA}"
|
|
54: test "$health" = "200" && test "${rc:-NA}" = "0" && return 0
|
|
63: crowdsec:
|
|
84: docker compose up -d crowdsec
|
|
91: crowdsec:
|
|
146:remove("/opt/stacks/crowdsec/config/config.yaml")
|
|
147:remove("/opt/stacks/crowdsec/config/user.yaml")
|
|
154:p = Path("/opt/stacks/crowdsec/config/config.yaml")
|
|
183: " credentials_path: /etc/crowdsec/online_api_credentials.yaml",
|
|
194:echo "NETBIRD_RESTART"
|
|
195:systemctl restart netbird 2>/dev/null || true
|
|
198:echo "NETBIRD_STATUS"
|
|
199:netbird status 2>&1 | redact || true
|
|
201:echo "PEERS_RELAY_MAIL_EDGE"
|
|
202:netbird status --json >/tmp/771z_nb.json 2>/tmp/771z_nb.err || true
|
|
211: ip=peer.get("netbirdIp","")
|
|
212: if ip in {"[IP]","[IP]","[IP]"} or any(x in fqdn for x in ["relay","mail","edge-vm"]):
|
|
216:echo "RELAY_MAIL_CONNECTIVITY"
|
|
224:echo "WG_ALLOWED_IPS_RELAY_MAIL"
|
|
226: wg show wt0 allowed-ips 2>/dev/null | sed -E 's/^[A-Za-z0-9+\/=]{20,}/PEERKEY_REDACTED/' | grep -E '100\.100\.19\.1|100\.100\.147\.204|0\.0\.0\.0/0|100\.100\.0\.0/16' || echo RELAY_MAIL_OR_DEFAULT_NOT_IN_ALLOWED_IPS
|
|
229:echo "CAPI_DIRECT_TEST_AFTER_MANUAL_NETBIRD"
|
|
230:capi_code="$(curl -4 -sk --http1.1 --connect-timeout 12 --max-time 45 -o /tmp/771z_capi.body -w '%{http_code}' https://api.crowdsec.net/v3/watchers/login || true)"
|
|
240: write_health "REVIEW_MANUAL_NETBIRD_EGRESS_NOT_READY" "After manual NetBird setup, edge still cannot reach api.crowdsec.net; CrowdSec remains LAPI-only no-capi"
|
|
241: echo "EDGE_STATUS=REVIEW_MANUAL_NETBIRD_EGRESS_NOT_READY_CAPI_NOT_DONE"
|
|
248:mkdir -p "manual-backups/771z-$TS"
|
|
DEPENDENCY_FILE|path=/srv/homelab-ops/kb/current/03_ACCESS_AND_SECRETS.md|mtime=2026-08-21 00:06:53.672037607 +0300|sha256=e750284217dc41b5809268a0bbc54ed0948bdd14680fe2f282f307640937496c
|
|
DEPENDENCY_FILE|path=/usr/local/sbin/homelab-alerting-health|mtime=2026-07-06 17:21:33.211361096 +0300|sha256=2050bac8c2ca8f1116e6d4fabe476292253bf38c56a49456242606980b4eb797
|
|
4:OUT="/var/lib/homelab-health/alerting.txt"
|
|
5:mkdir -p /var/lib/homelab-health
|
|
20:KUMA_STATE="$(ssh -n -o BatchMode=yes -o ConnectTimeout=8 "debian@$MON_HOST" "sudo docker inspect -f '{{.State.Status}} {{if .State.Health}}{{.State.Health.Status}}{{end}}' uptime-kuma" 2>/dev/null || true)"
|
|
23:GOTIFY_STATE="$(ssh -n -o BatchMode=yes -o ConnectTimeout=8 "debian@$MON_HOST" "sudo docker inspect -f '{{.State.Status}}' gotify" 2>/dev/null || true)"
|
|
24:if echo "$GOTIFY_STATE" | grep -q "running"; then ok gotify_container; else bad gotify_container; fi
|
|
26:MON_ROW="$(ssh -n -o BatchMode=yes -o ConnectTimeout=8 "debian@$MON_HOST" "sudo sqlite3 '$KUMA_DB' \"SELECT m.id || '|' || m.active || '|' || m.type || '|' || m.url || '|' || m.keyword || '|' || COALESCE(n.name,'') || '|' || COALESCE(n.active,'') FROM monitor m LEFT JOIN monitor_notification mn ON mn.monitor_id=m.id LEFT JOIN notification n ON n.id=mn.notification_id WHERE m.name='Homelab Overall Health' LIMIT 1;\"" 2>/dev/null || true)"
|
|
30:if [[ "${NNAME:-}" = "Gotify" && "${NACTIVE:-}" = "1" ]]; then ok gotify_link; else bad gotify_link; fi
|
|
33:echo "STATUS=$STATUS TS=$TS TYPE=alerting BAD=$BAD$DETAILS kuma_state=${KUMA_STATE// /_} gotify_state=${GOTIFY_STATE// /_} monitor=${MON_ROW:-missing}" > "$OUT"
|
|
DEPENDENCY_FILE|path=/usr/local/sbin/homelab-kuma-monitor-policy|mtime=2026-07-06 17:21:33.211264665 +0300|sha256=87e1f14bde3e5eb2d0512c949e1e32d7fe3214f38797559b6e3e1a9d855f059a
|
|
4:OUT="/var/lib/homelab-health/kuma-monitor-policy.txt"
|
|
12:PUBLIC_URL="https://backup.gram1.ru/overall.txt"
|
|
14:mkdir -p /var/lib/homelab-health "$ROOT"
|
|
26:SQL="SELECT id||'|'||name||'|'||type||'|'||url||'|'||COALESCE(keyword,'')||'|'||active FROM monitor WHERE name IN ('Homelab Overall Health','Homelab Overall Public Health','File Browser Backup','File Browser Offhost Backup','File Browser Restore Validation','NPMplus Kuma Config Backup','Paperless Restore') ORDER BY id;"
|
|
30:LINKSQL="SELECT m.name||'|'||COALESCE(n.name,'')||'|'||COALESCE(n.active,'') FROM monitor m LEFT JOIN monitor_notification mn ON mn.monitor_id=m.id LEFT JOIN notification n ON n.id=mn.notification_id WHERE m.name IN ('Homelab Overall Health','Homelab Overall Public Health') ORDER BY m.name,n.name;"
|
|
34:printf "%s\n" "$ROWS" | grep -Fq "Homelab Overall Health|keyword|$PRIVATE_URL|STATUS=OK|1" && ok private_monitor || bad private_monitor
|
|
35:printf "%s\n" "$ROWS" | grep -Fq "Homelab Overall Public Health|keyword|$PUBLIC_URL|STATUS=OK|1" && ok public_monitor || bad public_monitor
|
|
36:printf "%s\n" "$LINKS" | grep -Fxq "Homelab Overall Health|Gotify|1" && ok private_gotify || bad private_gotify
|
|
37:printf "%s\n" "$LINKS" | grep -Fxq "Homelab Overall Public Health|Gotify|1" && ok public_gotify || bad public_gotify
|
|
39:LEGACY_FOUND="$(printf "%s\n" "$ROWS" | awk -F'|' '$2=="File Browser Backup" || $2=="File Browser Offhost Backup" || $2=="File Browser Restore Validation" || $2=="NPMplus Kuma Config Backup" || $2=="Paperless Restore" {c++} END{print c+0}')"
|
|
40:LEGACY_ACTIVE="$(printf "%s\n" "$ROWS" | awk -F'|' '($2=="File Browser Backup" || $2=="File Browser Offhost Backup" || $2=="File Browser Restore Validation" || $2=="NPMplus Kuma Config Backup" || $2=="Paperless Restore") && $6=="1" {c++} END{print c+0}')"
|
|
43:KUMA_STATE="$(ssh -n -o BatchMode=yes -o ConnectTimeout=8 "$KUMA_HOST" "sudo docker inspect -f '{{.State.Status}} {{if .State.Health}}{{.State.Health.Status}}{{end}}' uptime-kuma" 2>/dev/null || true)"
|
|
DEPENDENCY_FILE|path=/usr/local/sbin/homelab-runbook-generate|mtime=2026-07-03 14:21:01.771069001 +0300|sha256=956056878b1636369f39f8d79a1c725ebde17ee319bbbb5d5b09f188d656b23e
|
|
7:health = Path("/var/lib/homelab-health/runbooks.txt")
|
|
8:state = Path("/var/lib/homelab-backup/apps")
|
|
12:health.parent.mkdir(parents=True, exist_ok=True)
|
|
32: f"## Current verified backup state\n"
|
|
39: f"- local_staging_deleted_after_two_cloud_verify: {d.get('local_backup_deleted', 'na')}\n\n"
|
|
43: f"jq . /var/lib/homelab-backup/apps/latest-{app}.json\n"
|
|
44: f"appbackupctl restore-check {app}\n"
|
|
50: f"homelab-backupctl audit\n"
|
|
53: f"homelab-overall-health\n"
|
|
73:index += "homelab-overall-health\n"
|
|
74:index += "homelab-backupctl audit\n"
|
|
85: "backupctl restore-check\n"
|
|
86: "appbackupctl restore-check safety-net-control-plane\n"
|
|
91: "cat /var/lib/homelab-health/overall.txt\n"
|
|
92: "cat /var/lib/homelab-health/alerting.txt\n"
|
|
93: "ssh -n debian@[IP] \"sudo docker ps --format '{{.Names}}|{{.Status}}' | egrep 'uptime-kuma|gotify'\"\n"
|
|
96: "If backup verification, dependency map, or alerting health is not OK, stop automation and escalate manually.\n"
|
|
108:health.write_text(line + "\n")
|
|
DEPENDENCY_FILE|path=/usr/local/sbin/homelab-vps-identity-audit|mtime=2026-06-24 16:00:07.135818676 +0300|sha256=b709691c52ae48432b70e197eb4810b6c374a05911cee3b9fdba4ecc67c76c3a
|
|
5:VPS_ALIAS="${VPS_ALIAS:-netbird.vpn.gram1.ru}"
|
|
6:H="/var/lib/homelab-health/vps-identity-audit.txt"
|
|
9:mkdir -p /var/lib/homelab-health
|
|
23:backup="$(find /mnt/staging/netbird-vps-backups/snapshots -maxdepth 2 -type f -name 'netbird-vps-backup.tgz' 2>/dev/null | sort | tail -n1 || true)"
|
|
24:backup_hostkeys=NO
|
|
25:backup_match=NO
|
|
27:if [ -n "$backup" ]; then
|
|
28: if tar -tzf "$backup" | grep -Eq 'ssh_host_.*_key\.pub$'; then
|
|
29: backup_hostkeys=YES
|
|
31: tar -xOzf "$backup" "$k" >> "$TMP/backup-hostkeys.pub" 2>/dev/null || true
|
|
32: done < <(tar -tzf "$backup" | grep -E 'ssh_host_.*_key\.pub$')
|
|
33: ssh-keygen -lf "$TMP/backup-hostkeys.pub" > "$TMP/backup-fingerprints.txt" 2>/dev/null || true
|
|
34: if grep -Fxf "$TMP/current-fingerprints.txt" "$TMP/backup-fingerprints.txt" >/dev/null 2>&1; then
|
|
35: backup_match=YES
|
|
41:[ "$backup_hostkeys" = "YES" ] && trust="BLOCKED_HOSTKEY_MISMATCH"
|
|
42:[ "$backup_match" = "YES" ] && trust="VERIFIED_BACKUP_MATCH"
|
|
46: "$status" "$TS" "$VPS_IP" "$VPS_ALIAS" "${alias_ip:-NONE}" "$trust" "$backup_hostkeys" > "$H"
|
|
DEPENDENCY_FILE|path=/var/lib/homelab-backup-matrix/compat/netbird_vps_backup--netbird-vps-trust.txt--44dfdc0d5d0a.txt|mtime=2026-07-18 01:52:29.872264457 +0300|sha256=c1f8ec2f8be329eba2bed981a6c281a2b5d932ae36e8440b07895ff7db93084a
|
|
3:TYPE=backup-matrix-generated
|
|
4:JOB_ID=netbird_vps_backup
|
|
5:JOB_NAME=netbird-vps-backup
|
|
7:TIMER=netbird-vps-backup.timer
|
|
8:SERVICE=netbird-vps-backup.service
|
|
11:SOURCE_KIND=health_file
|
|
12:EVIDENCE_LEVEL=artifact_or_health
|
|
15:SOURCE_HEALTH=/var/lib/homelab-backup-matrix/evidence/netbird_vps_backup/0-netbird-vps-trust.txt
|
|
21:LEGACY_SOURCE_PATH=/var/lib/homelab-health/netbird-vps-trust.txt
|
|
22:CANONICAL_JOB_ID=netbird_vps_backup
|
|
23:CANONICAL_STATE=/var/lib/homelab-backup-matrix/state.json
|
|
DEPENDENCY_FILE|path=/var/lib/homelab-backup-matrix/evidence/homelab_verified_backup_health/0-backup-registry-latest.txt|mtime=2026-07-24 06:46:41.644124478 +0300|sha256=606f956d0c95435b4547a71a7ed3c5d8d0461700437815d064e3a7363e19ef21
|
|
22:app=gotify job=homelab-pve01-app-gotify-20260724T014943Z status=success size=64413 deleted=true proxy_hosts=na certs=na
|
|
25:app=healthchecks job=homelab-pve01-app-healthchecks-20260724T025849Z status=success size=686 deleted=true proxy_hosts=na certs=na
|
|
53:Sat 2026-07-25 [IPV6] MSK 20h Fri 2026-07-24 [IPV6] MSK 3h 33min ago homelab-emergency-backup.timer homelab-emergency-backup.service
|
|
54:Sat 2026-07-25 [IPV6] MSK 21h Fri 2026-07-24 [IPV6] MSK 2h 58min ago homelab-appbackup-dockge-stacks.timer homelab-appbackup-dockge-stacks.service
|
|
55:Sat 2026-07-25 [IPV6] MSK 21h Fri 2026-07-24 [IPV6] MSK 2h 29min ago homelab-appbackup-npmplus-edge.timer homelab-appbackup-npmplus-edge.service
|
|
56:Sat 2026-07-25 [IPV6] MSK 21h Fri 2026-07-24 [IPV6] MSK 2h 0min ago homelab-appbackup-sqlite-batch.timer homelab-appbackup-sqlite-batch.service
|
|
57:Sat 2026-07-25 [IPV6] MSK 22h Fri 2026-07-24 [IPV6] MSK 1h 28min ago homelab-appbackup-postgres-batch.timer homelab-appbackup-postgres-batch.service
|
|
58:Sat 2026-07-25 [IPV6] MSK 22h Fri 2026-07-24 [IPV6] MSK 52min ago homelab-appbackup-mariadb-batch.timer homelab-appbackup-mariadb-batch.service
|
|
63:/var/lib/homelab-backup/quarantine/20260703T093243Z
|
|
DEPENDENCY_FILE|path=/var/lib/homelab-backup-matrix/evidence-map.json|mtime=2026-07-10 12:47:24.326276710 +0300|sha256=0735c335850b22ee4947e0940c6cfd1cc1525f340f102b7f5bcfef5069f165c4
|
|
6: "job": "filebrowser-backup",
|
|
7: "source_path": "/var/lib/homelab-health/filebrowser-backup.txt",
|
|
8: "evidence_path": "/var/lib/homelab-backup-matrix/evidence/filebrowser_backup/0-filebrowser-backup.txt"
|
|
12: "source_path": "/var/lib/homelab-health/filebrowser-restore.txt",
|
|
13: "evidence_path": "/var/lib/homelab-backup-matrix/evidence/filebrowser_restore_validate/0-filebrowser-restore.txt"
|
|
16: "job": "homelab-appbackup-dockge-stacks",
|
|
17: "source_path": "/var/lib/homelab-backup/apps/latest-dockge-stacks.json",
|
|
18: "evidence_path": "/var/lib/homelab-backup-matrix/evidence/homelab_appbackup_dockge_stacks/0-latest-dockge-stacks.json"
|
|
21: "job": "homelab-appbackup-npmplus-edge",
|
|
22: "source_path": "/var/lib/homelab-backup/apps/latest-npmplus-edge.json",
|
|
23: "evidence_path": "/var/lib/homelab-backup-matrix/evidence/homelab_appbackup_npmplus_edge/0-latest-npmplus-edge.json"
|
|
26: "job": "homelab-authentik-backup",
|
|
27: "source_path": "/var/lib/homelab-health/authentik-backup.txt",
|
|
28: "evidence_path": "/var/lib/homelab-backup-matrix/evidence/homelab_authentik_backup/0-authentik-backup.txt"
|
|
31: "job": "homelab-authentik-backup",
|
|
32: "source_path": "/var/lib/homelab-health/authentik-offhost.txt",
|
|
33: "evidence_path": "/var/lib/homelab-backup-matrix/evidence/homelab_authentik_backup/1-authentik-offhost.txt"
|
|
36: "job": "homelab-authentik-backup",
|
|
37: "source_path": "/var/lib/homelab-health/authentik-restore.txt",
|
|
38: "evidence_path": "/var/lib/homelab-backup-matrix/evidence/homelab_authentik_backup/2-authentik-restore.txt"
|
|
41: "job": "homelab-backup-coverage-matrix",
|
|
42: "source_path": "/var/lib/homelab-health/backup-coverage-matrix.txt",
|
|
43: "evidence_path": "/var/lib/homelab-backup-matrix/evidence/homelab_backup_coverage_matrix/0-backup-coverage-matrix.txt"
|
|
46: "job": "homelab-backup-sla-health",
|
|
47: "source_path": "/var/lib/homelab-health/backup-sla.txt",
|
|
48: "evidence_path": "/var/lib/homelab-backup-matrix/evidence/homelab_backup_sla_health/0-backup-sla.txt"
|
|
51: "job": "homelab-edge-vm-vzdump-backup",
|
|
52: "source_path": "/var/lib/homelab-health/edge-vm-vzdump-backup.txt",
|
|
53: "evidence_path": "/var/lib/homelab-backup-matrix/evidence/homelab_edge_vm_vzdump_backup/0-edge-vm-vzdump-backup.txt"
|
|
56: "job": "homelab-emergency-backup",
|
|
57: "source_path": "/var/lib/homelab-backup/latest-emergency-pack.json",
|
|
58: "evidence_path": "/var/lib/homelab-backup-matrix/evidence/homelab_emergency_backup/0-latest-emergency-pack.json"
|
|
61: "job": "homelab-extended-appbackup",
|
|
62: "source_path": "/var/lib/homelab-health/extended-appbackup.txt",
|
|
63: "evidence_path": "/var/lib/homelab-backup-matrix/evidence/homelab_extended_appbackup/0-extended-appbackup.txt"
|
|
66: "job": "homelab-gitea-backup",
|
|
DEPENDENCY_FILE|path=/var/lib/homelab-backup-matrix/evidence/netbird_vps_backup/0-netbird-vps-backup.txt|mtime=2026-07-01 04:31:05.433646905 +0300|sha256=372804695d6dba94e29e254f2175701e976692e7b15ee83fdcb6f10e05ca624e
|
|
2:SERVICE=netbird-vps-backup
|
|
5:SNAPSHOT=/mnt/staging/netbird-vps-backups/snapshots/20260701T013054Z
|
|
6:LATEST_SYMLINK=/mnt/staging/netbird-vps-backups/snapshots/latest
|
|
DEPENDENCY_FILE|path=/var/lib/homelab-backup-matrix/migration-plan.json|mtime=2026-07-10 13:10:05.080842534 +0300|sha256=260b6ef9d8c3370b6b429694ff642319ece1592f9e7c261e37b9fc3de0d33581
|
|
3: "dpkg-db-backup",
|
|
4: "filebrowser-backup",
|
|
6: "homelab-appbackup-dockge-stacks",
|
|
7: "homelab-appbackup-mariadb-batch",
|
|
8: "homelab-appbackup-npmplus-edge",
|
|
9: "homelab-appbackup-postgres-batch",
|
|
10: "homelab-appbackup-sqlite-batch",
|
|
11: "homelab-auto-backup",
|
|
12: "homelab-backup-coverage-matrix",
|
|
13: "homelab-backup-freshness",
|
|
14: "homelab-backup-healthcheck",
|
|
15: "homelab-backup-sla-health",
|
|
16: "homelab-edge-vm-vzdump-backup",
|
|
17: "homelab-emergency-backup",
|
|
18: "homelab-extended-appbackup",
|
|
19: "homelab-mail-cloud-restore-drill",
|
|
20: "homelab-verified-backup-health",
|
|
21: "homelab-vm-backup-health",
|
|
22: "homelab-vm-backup-policy",
|
|
23: "homelab-vm-full-backup",
|
|
27: "immich-media-backup",
|
|
28: "memos-backup",
|
|
29: "netbird-vps-backup",
|
|
30: "paperless-backup",
|
|
32: "skladchik-reports-monitor-backup-restore-check",
|
|
33: "skladchik-reports-monitor-backup"
|
|
38: "homelab-appbackup-postgres-batch"
|
|
40: "owner_job": "homelab-appbackup-postgres-batch",
|
|
41: "path": "/var/lib/homelab-health/authentik-backup.txt"
|
|
45: "homelab-appbackup-postgres-batch"
|
|
47: "owner_job": "homelab-appbackup-postgres-batch",
|
|
48: "path": "/var/lib/homelab-health/authentik-offhost.txt"
|
|
52: "homelab-appbackup-postgres-batch"
|
|
54: "owner_job": "homelab-appbackup-postgres-batch",
|
|
55: "path": "/var/lib/homelab-health/authentik-restore.txt"
|
|
59: "homelab-backup-coverage-matrix"
|
|
DEPENDENCY_FILE|path=/var/lib/homelab-backup-matrix/rendered/netbird-vps-backup.txt|mtime=2026-08-30 21:08:09.037130131 +0300|sha256=d2d09878e7ccd4159c2c66225fea6bbcf30d7ba9a7e70ab30f5df4fae631620c
|
|
3:TYPE=backup-matrix-generated
|
|
4:JOB_ID=netbird_vps_backup
|
|
5:JOB_NAME=netbird-vps-backup
|
|
7:TIMER=netbird-vps-backup.timer
|
|
8:SERVICE=netbird-vps-backup.service
|
|
11:SOURCE_KIND=health_file
|
|
12:EVIDENCE_LEVEL=artifact_or_health
|
|
15:SOURCE_HEALTH=/var/lib/homelab-backup-matrix/evidence/netbird_vps_backup/0-netbird-vps-backup.txt
|
|
DEPENDENCY_FILE|path=/var/lib/homelab-backup-matrix/state.json|mtime=2026-08-30 21:08:09.112843466 +0300|sha256=bba7caf3207a046105369da4d140f3342dec2e0897d91d5b4e2e099f5aab6b90
|
|
3: "compatibility_map": "/var/lib/homelab-backup-matrix/compat-map.json",
|
|
12: "generated_health_output": "/var/lib/homelab-backup-matrix/rendered/dpkg-db-backup.txt",
|
|
13: "id": "dpkg_db_backup",
|
|
15: "name": "dpkg-db-backup",
|
|
16: "service": "dpkg-db-backup.service",
|
|
19: "source_health": "systemd:dpkg-db-backup.service",
|
|
24: "timer": "dpkg-db-backup.timer",
|
|
32: "evidence_level": "artifact_or_health",
|
|
34: "generated_health_output": "/var/lib/homelab-backup-matrix/rendered/filebrowser-backup.txt",
|
|
35: "id": "filebrowser_backup",
|
|
37: "name": "filebrowser-backup",
|
|
38: "service": "filebrowser-backup.service",
|
|
41: "source_health": "/var/lib/homelab-backup/apps/latest-filebrowser.json",
|
|
42: "source_kind": "health_file",
|
|
46: "timer": "filebrowser-backup.timer",
|
|
54: "evidence_level": "artifact_or_health",
|
|
56: "generated_health_output": "/var/lib/homelab-backup-matrix/rendered/filebrowser-restore-validate.txt",
|
|
63: "source_health": "/var/lib/homelab-backup-matrix/evidence/filebrowser_restore_validate/0-filebrowser-restore.txt",
|
|
64: "source_kind": "health_file",
|
|
76: "evidence_level": "artifact_or_health",
|
|
78: "generated_health_output": "/var/lib/homelab-backup-matrix/rendered/homelab-appbackup-dockge-stacks.txt",
|
|
79: "id": "homelab_appbackup_dockge_stacks",
|
|
81: "name": "homelab-appbackup-dockge-stacks",
|
|
82: "service": "homelab-appbackup-dockge-stacks.service",
|
|
85: "source_health": "/var/lib/homelab-backup-matrix/evidence/homelab_appbackup_dockge_stacks/0-latest-dockge-stacks.json",
|
|
86: "source_kind": "health_file",
|
|
90: "timer": "homelab-appbackup-dockge-stacks.timer",
|
|
100: "generated_health_output": "/var/lib/homelab-backup-matrix/rendered/homelab-appbackup-mariadb-batch.txt",
|
|
101: "id": "homelab_appbackup_mariadb_batch",
|
|
103: "name": "homelab-appbackup-mariadb-batch",
|
|
104: "service": "homelab-appbackup-mariadb-batch.service",
|
|
107: "source_health": "systemd:homelab-appbackup-mariadb-batch.service",
|
|
112: "timer": "homelab-appbackup-mariadb-batch.timer",
|
|
120: "evidence_level": "artifact_or_health",
|
|
122: "generated_health_output": "/var/lib/homelab-backup-matrix/rendered/homelab-appbackup-npmplus-edge.txt",
|
|
123: "id": "homelab_appbackup_npmplus_edge",
|
|
DEPENDENCY_FILE_INDEX_END=1
|
|
RELAY_HEALTHCHECK_ARTIFACT_BEGIN=1
|
|
ARTIFACT_MISSING=/usr/local/sbin/pvepro-relay-healthcheck
|
|
ARTIFACT_MISSING=/etc/systemd/system/pvepro-relay-healthcheck.service
|
|
ARTIFACT_MISSING=/etc/systemd/system/pvepro-relay-healthcheck.timer
|
|
ARTIFACT_MISSING=/etc/systemd/system/pvepro-relay-healthcheck.service.d/20-gotify-notifier.conf
|
|
RELAY_HEALTHCHECK_ARTIFACT_END=1
|
|
NETBIRD_STATE_BEGIN=1
|
|
NETBIRD_STATUS_JSON_RC=0
|
|
NETBIRD_JSON_PARSE=true
|
|
NETBIRD_DAEMON_STATUS=Connected
|
|
NETBIRD_MANAGEMENT_CONNECTED=false
|
|
NETBIRD_SIGNAL_CONNECTED=false
|
|
NETBIRD_RELAYS_TOTAL=2
|
|
NETBIRD_RELAYS_AVAILABLE=0
|
|
NETBIRD_PEER|fqdn=pve02.netbird.selfhosted|status=Connected|connection=P2P
|
|
NETBIRD_PEER|fqdn=home-pc.netbird.selfhosted|status=Connecting|connection=-
|
|
NETBIRD_PEER|fqdn=relay.netbird.selfhosted|status=Connecting|connection=-
|
|
NETBIRD_PEER|fqdn=pve03.netbird.selfhosted|status=Connected|connection=P2P
|
|
NETBIRD_PEER|fqdn=edge-vm.netbird.selfhosted|status=Connecting|connection=-
|
|
NETBIRD_PEER|fqdn=desktop-moy.netbird.selfhosted|status=Connecting|connection=-
|
|
NETBIRD_PEER|fqdn=comp01.netbird.selfhosted|status=Connecting|connection=-
|
|
NETBIRD_PEER|fqdn=e3qxxx.netbird.selfhosted|status=Connecting|connection=-
|
|
NETBIRD_PEER|fqdn=mail.netbird.selfhosted|status=Connecting|connection=-
|
|
NETBIRD_PEER|fqdn=desktop-server.netbird.selfhosted|status=Connecting|connection=-
|
|
NETBIRD_PEER|fqdn=e3qxxx-183-106.netbird.selfhosted|status=Connecting|connection=-
|
|
NETBIRD_NETWORKS_LIST_RC=0
|
|
No networks available.
|
|
NETBIRD_ROUTES_LIST_RC=0
|
|
No networks available.
|
|
NETBIRD_STATE_END=1
|
|
PROCESS_AND_SOCKET_REFERENCES_BEGIN=1
|
|
1220 netbird /usr/bin/netbird service run --log-level info --daemon-addr unix:///var/run/netbird.sock --log-file /var/log/netbird/client.log
|
|
ESTAB 0 1500 [IP]:33490 [IP]:443 users:(("netbird",pid=1220,fd=35))
|
|
ESTAB 0 1500 [IP]:33632 [IP]:443 users:(("netbird",pid=1220,fd=18))
|
|
ESTAB 0 1500 [IP]:34548 [IP]:443 users:(("netbird",pid=1220,fd=43))
|
|
ESTAB 0 1500 [IP]:37588 [IP]:443 users:(("netbird",pid=1220,fd=19))
|
|
ESTAB 0 1500 [IP]:39034 [IP]:443 users:(("netbird",pid=1220,fd=20))
|
|
ESTAB 0 1500 [IP]:39134 [IP]:443 users:(("netbird",pid=1220,fd=40))
|
|
ESTAB 0 1500 [IP]:40402 [IP]:443 users:(("netbird",pid=1220,fd=11))
|
|
ESTAB 0 1500 [IP]:41518 [IP]:443 users:(("netbird",pid=1220,fd=33))
|
|
ESTAB 0 1500 [IP]:43330 [IP]:443 users:(("netbird",pid=1220,fd=17))
|
|
ESTAB 0 1500 [IP]:43952 [IP]:443 users:(("netbird",pid=1220,fd=31))
|
|
ESTAB 0 1500 [IP]:44564 [IP]:443 users:(("netbird",pid=1220,fd=41))
|
|
ESTAB 0 1500 [IP]:47572 [IP]:443 users:(("netbird",pid=1220,fd=37))
|
|
ESTAB 0 1500 [IP]:48242 [IP]:443 users:(("netbird",pid=1220,fd=30))
|
|
ESTAB 0 0 [IP]:22 [IP]:61241 users:(("sshd-session",pid=2731736,fd=7),("sshd-session",pid=2731520,fd=7))
|
|
ESTAB 0 1500 [IP]:50512 [IP]:443 users:(("netbird",pid=1220,fd=21))
|
|
ESTAB 0 1500 [IP]:52460 [IP]:443 users:(("netbird",pid=1220,fd=16))
|
|
ESTAB 0 1500 [IP]:53972 [IP]:443 users:(("netbird",pid=1220,fd=38))
|
|
ESTAB 0 1500 [IP]:59556 [IP]:443 users:(("netbird",pid=1220,fd=39))
|
|
PROCESS_AND_SOCKET_REFERENCES_END=1
|
|
DNS_AND_HOST_REFERENCES_BEGIN=1
|
|
DNS_FILE=/etc/hosts|sha256=c3b9101ad96911d55328b70e12e87f9b17e4ef89fce47dc619478e113c4855ec
|
|
11:[IP] pve01.gram1.ru pve01
|
|
12:[IP] pve02.gram1.ru pve02
|
|
13:[IP] pve03.gram1.ru pve03
|
|
DNS_FILE=/etc/resolv.conf|sha256=be701c9795e17f831d600a2ad82e50a15785fb48a21d105d6413bb5b98a79fef
|
|
1:# Generated by NetBird
|
|
2:# The original file can be restored from /etc/resolv.conf.original.netbird
|
|
4:search netbird.selfhosted gram1.ru
|
|
DNS_AND_HOST_REFERENCES_END=1
|
|
INTERPRETATION_RULES_BEGIN=1
|
|
RULE_1=DO_NOT_DELETE_PROVIDER_VPS_WHILE_MD_RELAY_BACKUP_OR_MONITORING_REFERENCE_IS_ACTIVE_OR_UNRECONCILED
|
|
RULE_2=DO_NOT_DELETE_WHILE_ANY_NETBIRD_ROUTE_OR_EXIT_NETWORK_STILL_SELECTS_MOLDOVA_FOR_AN_ACTIVE_PEER
|
|
RULE_3=DO_NOT_DELETE_WHILE_PVEPRO_RELAY_HEALTHCHECK_OR_ALERTING_EXPECTS_THIS_NODE_UNLESS_REPLACED_OR_RETIRED
|
|
RULE_4=DO_NOT_DELETE_WHILE_BACKUP_AUDIT_IS_FAILED_UNLESS_FAILURE_IS_PROVEN_UNRELATED_AND_BASELINE_IS_RESTORED
|
|
RULE_5=AFTER_DETACH_USE_A_REVERSIBLE_QUARANTINE_WINDOW_BEFORE_PROVIDER_DELETION
|
|
INTERPRETATION_RULES_END=1
|
|
DECISION=HOLD_DELETE_PENDING_EXACT_PREFLIGHT_REVIEW
|
|
NEXT_GATE=READ_EXACT_1130R1_AND_BUILD_NARROW_DETACH_OR_QUARANTINE_PLAN
|
|
TASK_RESULT=PASS_MOLDOVA_RETIRE_PREFLIGHT_EVIDENCE_COLLECTED
|
|
WORKERS2_MOLDOVA_RETIRE_PREFLIGHT_END=1
|
|
|
|
OUTPUT_END
|
|
CHAT_OUTPUT_END
|