homelab-admin/homelab-public-context
1
0
Fork 0
Code Activity
Files
491cb026a56ea19823f9288759d46d4ccb9a69d7
homelab-public-context/runtime/latest.json
T

39 lines
40 KiB
JSON
Raw Blame History

{
"schema_version": 1,
"channel": "homelab-runtime",
"command_id": "SUPPORT-260921-HOMELAB-BACKUP-CLOSURE-READINESS-READONLY-1039R1",
"status": "OK",
"rc": 0,
"host": "pve01",
"mode": "read-only",
"component": "homelab-backup-closure-readiness",
"started_at_utc": "2026-09-21T03:05:06Z",
"finished_at_utc": "2026-09-21T03:05:15Z",
"reference_register_checked": true,
"reference_sha256": "5a3d8e5154c41cb582a4e0aca68090be1f0138918bf82131a948df326f9d8d66",
"error_register_checked": true,
"error_register_sha256": "3b09a553ec0f527ed3afeed4753f52a74ee3036045bbfb3c685e9f8af4ba7ba0",
"command_sha256": "433bf07285750ded0b90a059ecc045f56dd0ae5ae66628debcaf9ed01b0b47b4",
"duplicate_failed_command_blocked": false,
"block_reason": null,
"execution_started": true,
"change_declared": false,
"result_contract_valid": true,
"result_contract_status": null,
"result_contract_error": null,
"command_rc": 0,
"changes_made": false,
"rollback_started": false,
"rollback_restored": null,
"mutation_outcome": "NO_MUTATION",
"sanitized": true,
"secrets_included": false,
"private_addresses_included": false,
"raw_evidence_retained_locally": true,
"raw_evidence_sha256": "ea37b24ec88205e6621f2dae60bc6d76bf9fdaea2563042d51d916a25a31d088",
"sanitized_output_sha256": "ea37b24ec88205e6621f2dae60bc6d76bf9fdaea2563042d51d916a25a31d088",
"output_truncated_in_json": false,
"full_sanitized_output_url": "https://git.gram1.ru/.well-known/homelab-runtime/latest.txt",
"output": "WORKERS2_BACKUP_CLOSURE_READINESS_BEGIN=1\nCOMMAND_ID=SUPPORT-260921-HOMELAB-BACKUP-CLOSURE-READINESS-READONLY-1039R1\nMODE=read-only\nCOMPONENT=homelab-backup-closure-readiness\nHOST=pve01\nUID=0\nMUTATION_BOUNDARY=READ_ONLY_PERSISTENT_STATE_NO_REPO_WRITE_NO_GIT_FETCH_NO_COMMIT_NO_PUSH_NO_SYSTEMD_ACTION_NO_BACKUP_NO_RESTORE_NO_RETENTION_DELETE_NO_CLOUD_DELETE_NO_MAIL;RETENTION_PLAN_USES_SELF_CLEANING_RUNTIME_TEMP_ONLY\nSOURCE_HEAD=50bf24f8ee27f3d2e0d1cbc6c17b62c69ff06bae\nSOURCE_WORKTREE_CLEAN=true\nACTIVE_REGISTRY=[{\"id\":\"filebrowser\",\"kind\":\"APP\",\"logical_id\":\"app-filebrowser\"},{\"id\":\"hbarhub\",\"kind\":\"APP\",\"logical_id\":\"app-hbarhub\"},{\"id\":\"immich\",\"kind\":\"APP\",\"logical_id\":\"app-immich\"},{\"id\":\"skladchik-reports-monitor\",\"kind\":\"APP\",\"logical_id\":\"app-skladchik-reports-monitor\"},{\"id\":\"control-bootstrap-v2\",\"kind\":\"CONTROL\",\"logical_id\":\"control-bootstrap-v2\"},{\"id\":\"110\",\"kind\":\"VM\",\"logical_id\":\"vm110\"},{\"id\":\"111\",\"kind\":\"VM\",\"logical_id\":\"vm111\"},{\"id\":\"112\",\"kind\":\"VM\",\"logical_id\":\"vm112\"},{\"id\":\"113\",\"kind\":\"VM\",\"logical_id\":\"vm113\"},{\"id\":\"130\",\"kind\":\"VM\",\"logical_id\":\"vm130\"},{\"id\":\"150\",\"kind\":\"VM\",\"logical_id\":\"vm150\"},{\"id\":\"160\",\"kind\":\"VM\",\"logical_id\":\"vm160\"},{\"id\":\"170\",\"kind\":\"VM\",\"logical_id\":\"vm170\"},{\"id\":\"171\",\"kind\":\"VM\",\"logical_id\":\"vm171\"},{\"id\":\"190\",\"kind\":\"VM\",\"logical_id\":\"vm190\"},{\"id\":\"9130\",\"kind\":\"VM\",\"logical_id\":\"vm9130\"}]\nACTIVE_REGISTRY_COUNT=16\nPRODUCTION_AUDIT_ALL={\"cloud_capacity_lines\":[\"CLOUD|remote=pve01-mail-01-crypt|status=OK|total_bytes=1099511627776|used_bytes=806303109447|free_bytes=293208518329\",\"CLOUD|remote=pve01-mail-02-crypt|status=OK|total_bytes=1099511627776|used_bytes=806303173306|free_bytes=293208454470\"],\"control_lines\":[\"CONTROL|state=STALE_PROVEN|CONTROL_FRESHNESS=STALE_PROVEN|generation=control-bootstrap-v2-pve01-20260916T120057Z|age_seconds=399837|sla_seconds=86400|archive_size_bytes=9002631|restore_evidence_checked=true|source_coverage=RESTORE_EVIDENCE|retention=ACTIVE_OWNER_AWARE\"],\"rc\":1,\"retention_lines\":[\"RETENTION|status=ACTIVE_OWNER_AWARE|historical_gc_proven=NO|owner_aware_gc_proven=YES|vm_keep=2|app_keep=7|control_keep=4\"],\"scheduler_lines\":[],\"source_lines\":[\"SOURCE|scope=local|label=VM_PROBE|status=OK|path=/usr/local/libexec/homelab-vm-cloud-quorum-freshness-probe|sha256=bd6257397c6ea910df2ea0dd57252b33bccd0bbd965175840486dfe5e079bf96\",\"SOURCE|scope=local|label=VM_WORKER|status=OK|path=/usr/local/libexec/homelab-vm-cloud-quorum-node-worker|sha256=91bdd116be865560a38105dffbd1a66d687232035e2359e4a5be4d225e9c61d1\",\"SOURCE|scope=local|label=APP_PROBE|status=OK|path=/usr/local/libexec/homelab-app-cloud-quorum-freshness-probe|sha256=9cd4d8f20ea449e16b3c8dc74a1b20a023ea609cf2404f5961dcd0489dc60987\",\"SOURCE|scope=local|label=APP_WORKER|status=OK|path=/usr/local/libexec/homelab-app-cloud-quorum-worker|sha256=1d6c4a04f762e2a33561c109b0c0a4878caaa48627903e7e78fdf5f22d79f196\",\"SOURCE|scope=local|label=APP_CORE|status=OK|path=/usr/local/libexec/homelab-app-cloud-quorum-core-helper|sha256=b7ea28e5f790bbaea140273e4c5dde63e505245448da119ee35011d0e99cb74e\",\"SOURCE|scope=local|label=TRANSPORT|status=OK|path=/usr/local/libexec/homelab-cloud-quorum-payload|sha256=df9e19485e5d87112bf58bf77bdddf3b9b896418dd85ff945a31cd5b6817f352\",\"SOURCE|scope=local|label=VM_QUEUE|status=DRIFT|path=/usr/local/sbin/homelab-vm-cloud-quorum-queue|sha256=fbc27ad67fc3d3f51934e1d73180fe0df8c9d15f129f34ebba39c56aa1d03612\",\"SOURCE|scope=local|label=APP_QUEUE|status=OK|path=/usr/local/sbin/homelab-app-cloud-quorum-queue|sha256=1f959049cca2710a2cb820cc1f6acab16399c6b6711b9e4522e0ecd49e0e4650\",\"SOURCE|scope=local|label=TARGET_REGISTRY_HELPER|status=OK|path=/usr/local/libexec/homelab-backup-target-registry|sha256=c681c44be7557d192d7b453b67e31164237874596168f1d4c78e1e926e2e9cb4\",\"SOURCE|scope=local|label=DISPATCHER|status=OK|path=/usr/local/sbin/homelab-backup-dispatcher|sha256=9752efc418bee24b876b6bc8c7ff946c605b027d71500bad4a9684ac0b82621c\",\"SOURCE|scope=local|label=CONTROL_BACKUP|status=OK|path=/usr/local/sbin/homelab-control-backup|sha256=0b3a0f685d4e67cb86250df89a4918aff2e6c67799b7d35b0e54b1bbc016ee57\",\"SOURCE|scope=local|label=CONTROL_BUILDER|status=OK|path=/usr/local/libexec/homelab-control-bundle-builder|sha256=2e1c3935b5016c09aad21f34a36f7e1f53188bbc34a3f033e59fe964571ec3f0\",\"SOURCE|scope=local|label=CONTROL_TIMER|status=OK|path=/etc/systemd/system/homelab-control-backup.timer|sha256=7c4136da20b9cd9162293380f5794d93e2c6099ce7d87ec00e0e7e19b6cd44a3\",\"SOURCE|scope=local|label=CONTROL_SERVICE|status=OK|path=/etc/systemd/system/homelab-control-backup.service|sha256=428d3e1f3042c3347b7c34b6f190e44fa35f7a97264eec3974239f581c9b9250\",\"SOURCE|scope=local|label=CONTROL_RUNTIME_MANIFEST|status=OK|path=/etc/homelab-backup/control-runtime-paths.txt|sha256=6777f95159db0fa213017f286f2cee6cc51e9963a01a0578cee8586e8731b2be\",\"SOURCE|scope=local|label=CONTROL_SYSTEMD_MANIFEST|status=OK|path=/etc/homelab-backup/control-systemd-paths.txt|sha256=7f990d6257b43f97c3c8407945ca9dae6113dc9064c6cf1f25e7fcdaf99d5163\",\"SOURCE|scope=local|label=VM_TIMER|status=OK|path=/etc/systemd/system/homelab-vm-cloud-quorum-queue.timer|sha256=9a78c9ea507116cbb9f5645fe697e34964c905b017cc98b17977cd5006aa5050\",\"SOURCE|scope=local|label=APP_TIMER|status=OK|path=/etc/systemd/system/homelab-app-cloud-quorum-queue.timer|sha256=af2eedce3df53029b591d31715a3f3e6688ddb997b4823f487e70c339ce88078\",\"SOURCE|scope=local|label=VM_SERVICE|status=OK|path=/etc/systemd/system/homelab-vm-cloud-quorum-queue.service|sha256=6aa2c72d3eda795f53d25292a11225ae13b418651cef9cce040564ee873fe759\",\"SOURCE|scope=local|label=APP_SERVICE|status=OK|path=/etc/systemd/system/homelab-app-cloud-quorum-queue.service|sha256=70b78ba529c270ec61d26a123d656385a8b2e2e4206ccf623111fa6ed3854602\",\"SOURCE|scope=local|label=CR0031_SEAL|status=OK|path=/var/lib/homelab-admin/seals/backup-vm-restore-evidence-hardening-v1.json|sha256=5f1274624374adbbb495b1701fe1eedaab028fad0f97a2409f0b79df9a791f9a\",\"SOURCE|scope=local|label=OBS_HELPER|status=OK|path=/usr/local/libexec/homelab-backup-observability|sha256=16dd24c8c84fe9bb3970a6cb7cd0e58dfa5243e9056f9f40f7785b8e61443581\",\"SOURCE|scope=local|label=OBS_CONFIG|status=OK|path=/etc/homelab-backup/observability.json|sha256=5b29bd9b76077a38f55ad2944efccdf8312c2d7dbfc8724cf54cfa81f8a9df3b\",\"SOURCE|scope=pve02|label=VM_PROBE|status=OK|path=/usr/local/libexec/homelab-vm-cloud-quorum-freshness-probe|sha256=bd6257397c6ea910df2ea0dd57252b33bccd0bbd965175840486dfe5e079bf96\",\"SOURCE|scope=pve02|label=VM_WORKER|status=OK|path=/usr/local/libexec/homelab-vm-cloud-quorum-node-worker|sha256=91bdd116be865560a38105dffbd1a66d687232035e2359e4a5be4d225e9c61d1\",\"SOURCE|scope=pve02|label=TRANSPORT|status=OK|path=/usr/local/libexec/homelab-cloud-quorum-payload|sha256=df9e19485e5d87112bf58bf77bdddf3b9b896418dd85ff945a31cd5b6817f352\",\"SOURCE|scope=pve03|label=VM_PROBE|status=OK|path=/usr/local/libexec/homelab-vm-cloud-quorum-freshness-probe|sha256=bd6257397c6ea910df2ea0dd57252b33bccd0bbd965175840486dfe5e079bf96\",\"SOURCE|scope=pve03|label=VM_WORKER|status=OK|path=/usr/local/libexec/homelab-vm-cloud-quorum-node-worker|sha256=91bdd116be865560a38105dffbd1a66d687232035e2359e4a5be4d225e9c61d1\",\"SOURCE|scope=pve03|label=TRANSPORT|status=OK|path=/usr/local/libexec/homelab-cloud-quorum-payload|sha256=df9e19485e5d87112bf58bf77bdddf3b9b896418dd85ff945a31cd5b6817f352\"],\"stderr_lines\":[],\"stdout_sha256\":\"c13e3e3163ecb076c059727f0ef06f50a48decba5e7d1e3f5a4f0ad7a304781a\",\"summary_lines\":[\"SUMMARY|AUDIT_STATUS=OK|BACKUP_STATUS=RED|RISK_SEVERITY=CRITICAL|RISK_REASONS=SOURCE_DRIFT,INFRASTRUCTURE_INVARIANT,RPO_BREACH,CAPACITY_CRITICAL|DR_PROOF_STATUS=PROVEN|CONTROL_FRESHNESS=STALE_PROVEN|LIFECYCLE_STATUS=CLOSED_PROVEN|VM_COUNT=11|APP_COUNT=4|FRESH=0|STALE=15|RESUME=0|UNVERIFIED=0|CONTROL_STATE=STALE_PROVEN|CLOUD_OK=2|SCHEDULER_OK=6|SOURCE_OK=28|SOURCE_DRIFT=1|STAGING_BAD=0|STAGING_WARN=0|PAYLOAD_LEAKS=0|RETENTION_STATUS=ACTIVE_OWNER_AWARE|OBSERVABILITY=ATTENTION|SIZE_ANOMALIES=0|CAPACITY=CRIT|GLOBAL_DR_PROVEN=YES\"],\"target_lines\":[\"TARGET|kind=VM|id=110|owner=pve01|state=STALE|generation=vm110-pve01-20260904T053821Z|age_seconds=1459532|sla_seconds=72000|archive_size_bytes=478357303|restore_evidence=/var/lib/homelab-backup/vm-restore-validation/vm110/vm110-pve01-20260904T053821Z.json\",\"TARGET|kind=VM|id=111|owner=pve02|state=STALE|generation=vm111-pve02-20260903T185424Z|age_seconds=1498111|sla_seconds=72000|archive_size_bytes=369892675|restore_evidence=/var/lib/homelab-backup/vm-restore-validation/vm111/vm111-pve02-20260903T185424Z.json\",\"TARGET|kind=VM|id=112|owner=pve01|state=STALE|generation=vm112-pve01-20260903T185707Z|age_seconds=1498025|sla_seconds=72000|archive_size_bytes=223672749|restore_evidence=/var/lib/homelab-backup/vm-restore-validation/vm112/vm112-pve01-20260903T185707Z.json\",\"TARGET|kind=VM|id=113|owner=pve02|state=STALE|generation=vm113-pve02-20260903T185827Z|age_seconds=1497945|sla_seconds=72000|archive_size_bytes=221529138|restore_evidence=/var/lib/homelab-backup/vm-restore-validation/vm113/vm113-pve02-20260903T185827Z.json\",\"TARGET|kind=VM|id=130|owner=pve03|state=STALE|generation=vm130-pve03-20260916T105207Z|age_seconds=401981|sla_seconds=72000|archive_size_bytes=10723620588|restore_evidence=/var/lib/homelab-backup/vm-restore-validation/vm130/vm130-pve03-20260916T105207Z.json\",\"TARGET|kind=VM|id=150|owner=pve01|state=STALE|generation=vm150-pve01-20260904T053517Z|age_seconds=1459645|sla_seconds=72000|archive_size_bytes=887338816|restore_evidence=/var/lib/homelab-backup/vm-restore-validation/vm150/vm150-pve01-20260904T053517Z.json\",\"TARGET|kind=VM|id=160|owner=pve02|state=STALE|generation=vm160-pve02-20260916T023620Z|age_seconds=433118|sla_seconds=72000|archive_size_bytes=3673704465|restore_evidence=/var/lib/homelab-backup/vm-restore-validation/vm160/vm160-pve02-20260916T023620Z.json\",\"TARGET|kind=VM|id=170|owner=pve01|state=STALE|generation=vm170-pve01-20260916T103019Z|age_seconds=404271|sla_seconds=72000|archive_size_bytes=7119997483|restore_evidence=/var/lib/homelab-backup/vm-restore-validation/vm170/vm170-pve01-20260916T103019Z.json\",\"TARGET|kind=VM|id=171|owner=pve01|state=STALE|generation=vm171-pve01-20260916T023018Z|age_seconds=433803|sla_seconds=72000|archive_size_bytes=2041990920|restore_evidence=/var/lib/homelab-backup/vm-restore-validation/vm171/vm171-pve01-20260916T023018Z.json\",\"TARGET|kind=VM|id=190|owner=pve03|state=STALE|generation=vm190-pve03-20260903T175951Z|age_seconds=1501230|sla_seconds=72000|archive_size_bytes=1557482946|restore_evidence=/var/lib/homelab-backup/vm-restore-validation/vm190/vm190-pve03-20260903T175951Z.json\",\"TARGET|kind=VM|id=9130|owner=pve02|state=STALE|generation=vm9130-pve02-20260903T180554Z|age_seconds=1498719|sla_seconds=72000|archive_size_bytes=10998362942|restore_evidence=/var/lib/homelab-backup/vm-restore-validation/vm9130/vm9130-pve02-20260903T180554Z.json\",\"TARGET|kind=APP|id=filebrowser|owner=pve01|state=STALE|generation=app-filebrowser-pve01-20260916T113558Z|age_seconds=401333|sla_seconds=72000|archive_size_bytes=3182|restore_evidence=/var/lib/homelab-backup/app-restore-validation/app-filebrowser/app-filebrowser-pve01-20260916T113558Z.json\",\"TARGET|kind=APP|id=skladchik-reports-monitor|owner=pve01|state=STALE|generation=app-skladchik-reports-monitor-pve01-20260916T113630Z|age_seconds=401306|sla_seconds=72000|archive_size_bytes=174001|restore_evidence=/var/lib/homelab-backup/app-restore-validation/app-skladchik-reports-monitor/app-skladchik-reports-monitor-pve01-20260916T113630Z.json\",\"TARGET|kind=APP|id=immich|owner=pve01|state=STALE|generation=app-immich-pve01-20260916T113650Z|age_seconds=401248|sla_seconds=72000|archive_size_bytes=270193610|restore_evidence=/var/lib/homelab-backup/app-restore-validation/app-immich/app-immich-pve01-20260916T113650Z.json\",\"TARGET|kind=APP|id=hbarhub|owner=pve01|state=STALE|generation=app-hbarhub-pve01-20260916T113814Z|age_seconds=401142|sla_seconds=72000|archive_size_bytes=77065509|restore_evidence=/var/lib/homelab-backup/app-restore-validation/app-hbarhub/app-hbarhub-pve01-20260916T113814Z.json\"]}\nCANONICAL_ACTIVE_COVERAGE=[{\"age_seconds\":401333,\"audit_record_present\":true,\"fresh\":false,\"generation\":\"app-filebrowser-pve01-20260916T113558Z\",\"id\":\"filebrowser\",\"kind\":\"APP\",\"logical_id\":\"app-filebrowser\",\"owner\":\"pve01\",\"raw_sha256\":\"216f5b973035d45c9a3004e021b0b66f463ccbb21164de1937812db6af4eccaf\",\"reason\":null,\"sla_seconds\":72000,\"state\":\"STALE\"},{\"age_seconds\":401142,\"audit_record_present\":true,\"fresh\":false,\"generation\":\"app-hbarhub-pve01-20260916T113814Z\",\"id\":\"hbarhub\",\"kind\":\"APP\",\"logical_id\":\"app-hbarhub\",\"owner\":\"pve01\",\"raw_sha256\":\"20f89567a3ac9c4757cb7200db1fc88d467ef861140bb697ee69283639251b03\",\"reason\":null,\"sla_seconds\":72000,\"state\":\"STALE\"},{\"age_seconds\":401248,\"audit_record_present\":true,\"fresh\":false,\"generation\":\"app-immich-pve01-20260916T113650Z\",\"id\":\"immich\",\"kind\":\"APP\",\"logical_id\":\"app-immich\",\"owner\":\"pve01\",\"raw_sha256\":\"63d7f7de67988de8e37c18abab531440ccddc58e3e7ac0b8100d4edf47505ec0\",\"reason\":null,\"sla_seconds\":72000,\"state\":\"STALE\"},{\"age_seconds\":401306,\"audit_record_present\":true,\"fresh\":false,\"generation\":\"app-skladchik-reports-monitor-pve01-20260916T113630Z\",\"id\":\"skladchik-reports-monitor\",\"kind\":\"APP\",\"logical_id\":\"app-skladchik-reports-monitor\",\"owner\":\"pve01\",\"raw_sha256\":\"a3b69f51d8caeffdc8fc5f0b5524f0fe9561e9a94ff6dfec625a9e0e09d6a73c\",\"reason\":null,\"sla_seconds\":72000,\"state\":\"STALE\"},{\"age_seconds\":399837,\"audit_record_present\":true,\"fresh\":false,\"generation\":\"control-bootstrap-v2-pve01-20260916T120057Z\",\"id\":\"control-bootstrap-v2\",\"kind\":\"CONTROL\",\"logical_id\":\"control-bootstrap-v2\",\"raw_sha256\":\"792a46d8c6ba7e805bab0547998e28bdcd9b8ddffb802db23874c690fdc28c7f\",\"sla_seconds\":86400,\"state\":\"STALE_PROVEN\"},{\"age_seconds\":1459532,\"audit_record_present\":true,\"fresh\":false,\"generation\":\"vm110-pve01-20260904T053821Z\",\"id\":\"110\",\"kind\":\"VM\",\"logical_id\":\"vm110\",\"owner\":\"pve01\",\"raw_sha256\":\"08fa7da3417c0f15987c9f9f8dbbe5fea5d304c99e4a93dde5a725fe2f7d5bbb\",\"reason\":null,\"sla_seconds\":72000,\"state\":\"STALE\"},{\"age_seconds\":1498111,\"audit_record_present\":true,\"fresh\":false,\"generation\":\"vm111-pve02-20260903T185424Z\",\"id\":\"111\",\"kind\":\"VM\",\"logical_id\":\"vm111\",\"owner\":\"pve02\",\"raw_sha256\":\"d4c82db9b6c611edad0b763bad1030e0b3688fba17d6d0b4fb304c3466c2e516\",\"reason\":null,\"sla_seconds\":72000,\"state\":\"STALE\"},{\"age_seconds\":1498025,\"audit_record_present\":true,\"fresh\":false,\"generation\":\"vm112-pve01-20260903T185707Z\",\"id\":\"112\",\"kind\":\"VM\",\"logical_id\":\"vm112\",\"owner\":\"pve01\",\"raw_sha256\":\"a3dd4c610f82eaf6d9bed57782993ea5b62d8c75922ff29caa6fd7af243ee9fe\",\"reason\":null,\"sla_seconds\":72000,\"state\":\"STALE\"},{\"age_seconds\":1497945,\"audit_record_present\":true,\"fresh\":false,\"generation\":\"vm113-pve02-20260903T185827Z\",\"id\":\"113\",\"kind\":\"VM\",\"logical_id\":\"vm113\",\"owner\":\"pve02\",\"raw_sha256\":\"5c181bb47c758f26118079401aab3f3d7bc7c144b1c41281a3ab355827c03ada\",\"reason\":null,\"sla_seconds\":72000,\"state\":\"STALE\"},{\"age_seconds\":401981,\"audit_record_present\":true,\"fresh\":false,\"generation\":\"vm130-pve03-20260916T105207Z\",\"id\":\"130\",\"kind\":\"VM\",\"logical_id\":\"vm130\",\"owner\":\"pve03\",\"raw_sha256\":\"588615fcd71793fdc9f8cfb767b1e614520efb69901c7f0fa2cbdd43c833d198\",\"reason\":null,\"sla_seconds\":72000,\"state\":\"STALE\"},{\"age_seconds\":1459645,\"audit_record_present\":true,\"fresh\":false,\"generation\":\"vm150-pve01-20260904T053517Z\",\"id\":\"150\",\"kind\":\"VM\",\"logical_id\":\"vm150\",\"owner\":\"pve01\",\"raw_sha256\":\"71afb5ff2f458d1d9d266ae2fa1504bfd623e4d0641ed13398227b12c3754c82\",\"reason\":null,\"sla_seconds\":72000,\"state\":\"STALE\"},{\"age_seconds\":433118,\"audit_record_present\":true,\"fresh\":false,\"generation\":\"vm160-pve02-20260916T023620Z\",\"id\":\"160\",\"kind\":\"VM\",\"logical_id\":\"vm160\",\"owner\":\"pve02\",\"raw_sha256\":\"94b44e839bc57ff3a3703d1da37eccf4b6817651aba3f2cafcf38f17d3174a1b\",\"reason\":null,\"sla_seconds\":72000,\"state\":\"STALE\"},{\"age_seconds\":404271,\"audit_record_present\":true,\"fresh\":false,\"generation\":\"vm170-pve01-20260916T103019Z\",\"id\":\"170\",\"kind\":\"VM\",\"logical_id\":\"vm170\",\"owner\":\"pve01\",\"raw_sha256\":\"2d757852c7af46fff5cdefc1c9da4867e95e241214548060932fbdfda0ba8fc4\",\"reason\":null,\"sla_seconds\":72000,\"state\":\"STALE\"},{\"age_seconds\":433803,\"audit_record_present\":true,\"fresh\":false,\"generation\":\"vm171-pve01-20260916T023018Z\",\"id\":\"171\",\"kind\":\"VM\",\"logical_id\":\"vm171\",\"owner\":\"pve01\",\"raw_sha256\":\"0c53fbd68b3c489f4692f739a4b4730d7ec441b16c38e5ae8dccb4e5cdd269be\",\"reason\":null,\"sla_seconds\":72000,\"state\":\"STALE\"},{\"age_seconds\":1501230,\"audit_record_present\":true,\"fresh\":false,\"generation\":\"vm190-pve03-20260903T175951Z\",\"id\":\"190\",\"kind\":\"VM\",\"logical_id\":\"vm190\",\"owner\":\"pve03\",\"raw_sha256\":\"2760651b740722b0d3f114160dd76e5f5d64417eff219ed77ca4ef1985617234\",\"reason\":null,\"sla_seconds\":72000,\"state\":\"STALE\"},{\"age_seconds\":1498719,\"audit_record_present\":true,\"fresh\":false,\"generation\":\"vm9130-pve02-20260903T180554Z\",\"id\":\"9130\",\"kind\":\"VM\",\"logical_id\":\"vm9130\",\"owner\":\"pve02\",\"raw_sha256\":\"a27dd550eb9a714b4ea019e654008f9b7fcb746dc163afc6f00d66037bb495bc\",\"reason\":null,\"sla_seconds\":72000,\"state\":\"STALE\"}]\nCANONICAL_ACTIVE_COVERAGE_BAD=[\"app-filebrowser\",\"app-hbarhub\",\"app-immich\",\"app-skladchik-reports-monitor\",\"control-bootstrap-v2\",\"vm110\",\"vm111\",\"vm112\",\"vm113\",\"vm130\",\"vm150\",\"vm160\",\"vm170\",\"vm171\",\"vm190\",\"vm9130\"]\nCLUSTER_RESOURCE_MAP={\"100\":{\"name\":\"pvepro-prod\",\"node\":\"pve01\",\"status\":\"running\",\"type\":\"qemu\"},\"101\":{\"name\":\"pvepro-staging-v1\",\"node\":\"pve01\",\"status\":\"stopped\",\"type\":\"qemu\"},\"110\":{\"name\":\"dns1\",\"node\":\"pve01\",\"status\":\"running\",\"type\":\"lxc\"},\"111\":{\"name\":\"dns2\",\"node\":\"pve02\",\"status\":\"running\",\"type\":\"lxc\"},\"112\":{\"name\":\"unbound1\",\"node\":\"pve01\",\"status\":\"running\",\"type\":\"lxc\"},\"113\":{\"name\":\"unbound2\",\"node\":\"pve02\",\"status\":\"running\",\"type\":\"lxc\"},\"130\":{\"name\":\"edge-vm\",\"node\":\"pve03\",\"status\":\"running\",\"type\":\"qemu\"},\"150\":{\"name\":\"snikket\",\"node\":\"pve01\",\"status\":\"running\",\"type\":\"qemu\"},\"160\":{\"name\":\"forum-prod\",\"node\":\"pve02\",\"status\":\"running\",\"type\":\"qemu\"},\"170\":{\"name\":\"core-apps\",\"node\":\"pve01\",\"status\":\"running\",\"type\":\"qemu\"},\"171\":{\"name\":\"monitoring\",\"node\":\"pve01\",\"status\":\"running\",\"type\":\"qemu\"},\"190\":{\"name\":\"homelab-ops-worker\",\"node\":\"pve03\",\"status\":\"running\",\"type\":\"qemu\"},\"210\":{\"name\":\"newfi-prod\",\"node\":\"pve01\",\"status\":\"running\",\"type\":\"qemu\"},\"211\":{\"name\":\"newfi-staging\",\"node\":\"pve03\",\"status\":\"running\",\"type\":\"qemu\"},\"9130\":{\"name\":\"edge-cold-standby\",\"node\":\"pve02\",\"status\":\"stopped\",\"type\":\"qemu\"}}\nCLOUD_B_RESTORE_CANDIDATES=[]\nCLOUD_B_RESTORE_PREFERRED=null\nRESTORE_INTERFACES=[{\"bytes\":2331,\"interface_lines\":[{\"line\":22,\"text\":\"ROOT=\\\"/var/tmp/homelab-app-restore-check/${JOB}\\\"\"},{\"line\":23,\"text\":\"LOGBASE=\\\"/tmp/app-restore-${APP//[^a-zA-Z0-9_-]/_}.$$\\\"\"},{\"line\":25,\"text\":\"trap \\\"echo \\\\\\\"ERROR app restore-check failed; kept: $ROOT\\\\\\\"; exit 1\\\" ERR\"},{\"line\":26,\"text\":\"echo \\\"restore-check app=$APP job=$JOB schema=$SCHEMA\\\"\"},{\"line\":27,\"text\":\"rclone copy \\\"$RA\\\" \\\"$ROOT/a\\\" --transfers=4 --checkers=8 --retries=3 --low-level-retries=10 >\\\"${LOGBASE}-a.log\\\" 2>&1\"},{\"line\":30,\"text\":\"(cd \\\"$ROOT/a\\\" && sha256sum -c checksums.sha256 >\\\"${LOGBASE}-a-sha.log\\\" && ARCHIVE=\\\"$(jq -r \\\".archive_name\\\" manifest.json)\\\" && tar -tzf \\\"$ARCHIVE\\\" >/dev/null && jq -e --arg job \\\"$JOB\\\" \\\".job_id == \\\\$job\\\" manifest.json >/dev/null)\"},{\"line\":31,\"text\":\"echo \\\"OK cloud A app restore-check\\\"\"},{\"line\":32,\"text\":\"rclone copy \\\"$RB\\\" \\\"$ROOT/b\\\" --transfers=4 --checkers=8 --retries=3 --low-level-retries=10 >\\\"${LOGBASE}-b.log\\\" 2>&1\"},{\"line\":35,\"text\":\"(cd \\\"$ROOT/b\\\" && sha256sum -c checksums.sha256 >\\\"${LOGBASE}-b-sha.log\\\" && ARCHIVE=\\\"$(jq -r \\\".archive_name\\\" manifest.json)\\\" && tar -tzf \\\"$ARCHIVE\\\" >/dev/null && jq -e --arg job \\\"$JOB\\\" \\\".job_id == \\\\$job\\\" manifest.json >/dev/null)\"},{\"line\":36,\"text\":\"echo \\\"OK cloud B app restore-check\\\"\"},{\"line\":38,\"text\":\"rmdir /var/tmp/homelab-app-restore-check 2>/dev/null || true\"},{\"line\":41,\"text\":\"echo \\\"OK app restore-check both clouds\\\"\"}],\"path\":\"/usr/local/sbin/appbackup-restore-check\",\"sha256\":\"c3f052c8ddba5304de7dd9ab0b03dc033a0d711c36d5c8b4128310e5cea86bcb\"},{\"bytes\":1437,\"interface_lines\":[{\"line\":8,\"text\":\"ROOT=\\\"/var/tmp/homelab-restore-check/${JOB}\\\"\"},{\"line\":10,\"text\":\"trap 'echo \\\"ERROR restore-check failed; kept: \\\"; exit 1' ERR\"},{\"line\":11,\"text\":\"echo \\\"restore-check job=$JOB\\\"\"},{\"line\":12,\"text\":\"rclone copy \\\"$RA\\\" \\\"$ROOT/a\\\" --transfers=4 --checkers=8 --retries=3 --low-level-retries=10 >/tmp/homelab-restore-a.log 2>&1; homelab-rehydrate-chunked-dir \\\"$ROOT/a\\\" >>/tmp/homelab-restore-a.log 2>&1\"},{\"line\":13,\"text\":\"(cd \\\"$ROOT/a\\\" && sha256sum -c checksums.sha256 >/tmp/homelab-restore-a-sha.log && ARCHIVE=\\\"$(jq -r .archive_name manifest.json)\\\" && tar -tzf \\\"$ARCHIVE\\\" >/dev/null && jq -e --arg job \\\"$JOB\\\" \\\".job_id == \\\\$job\\\" manifest.json >/dev/null)\"},{\"line\":14,\"text\":\"echo \\\"OK cloud A restore-check\\\"\"},{\"line\":15,\"text\":\"rclone copy \\\"$RB\\\" \\\"$ROOT/b\\\" --transfers=4 --checkers=8 --retries=3 --low-level-retries=10 >/tmp/homelab-restore-b.log 2>&1; homelab-rehydrate-chunked-dir \\\"$ROOT/b\\\" >>/tmp/homelab-restore-b.log 2>&1\"},{\"line\":16,\"text\":\"(cd \\\"$ROOT/b\\\" && sha256sum -c checksums.sha256 >/tmp/homelab-restore-b-sha.log && ARCHIVE=\\\"$(jq -r .archive_name manifest.json)\\\" && tar -tzf \\\"$ARCHIVE\\\" >/dev/null && jq -e --arg job \\\"$JOB\\\" \\\".job_id == \\\\$job\\\" manifest.json >/dev/null)\"},{\"line\":17,\"text\":\"echo \\\"OK cloud B restore-check\\\"\"},{\"line\":20,\"text\":\"echo \\\"OK restore-check both clouds\\\"\"}],\"path\":\"/usr/local/sbin/backup-restore-check\",\"sha256\":\"fc927d8f3b70029ed09c3074a29b9537a8e59ff6b09d4aedc1e8bce932a51446\"},{\"bytes\":7995,\"interface_lines\":[{\"line\":4,\"text\":\"LOCK=/run/homelab-mail-cloud-restore-drill.lock\"},{\"line\":5,\"text\":\"HEALTH=/var/lib/homelab-health/mail-cloud-restore-drill.txt\"},{\"line\":11,\"text\":\"WORK=/mnt/staging/mail-cloud-restore-drills/$TS\"},{\"line\":23,\"text\":\" echo \\\"STATUS=ERROR TS=$TS TYPE=mail-cloud-restore-drill REASON=lock_busy SECRET_PRINTED=no NOTES=lock_busy\\\" > \\\"$HEALTH\\\"\"},{\"line\":35,\"text\":\"# --- Critical/small restore drill ---\"},{\"line\":52,\"text\":\" homelab-rclone-cat-chunked \\\"${R}:$CRIT_DEST/$CRIT_BUNDLE\\\" > \\\"$OUT\\\" 2>\\\"$WORK/logs/${SAFE}_crit_download.err\\\"\"},{\"line\":69,\"text\":\" grep -q 'components/MANIFEST.txt' \\\"$WORK/logs/${SAFE}_crit_tar_list.txt\\\"\"},{\"line\":70,\"text\":\" MANIFEST_RC=$?\"},{\"line\":78,\"text\":\" [ \\\"$MANIFEST_RC\\\" = \\\"0\\\" ] || R_STATUS=ERROR\"},{\"line\":87,\"text\":\" NOTES=critical_restore_failed\"},{\"line\":90,\"text\":\" echo \\\"$R|$R_STATUS|DOWNLOAD=$DL_RC|SHA_MATCH=$SHA_MATCH|SIZE_MATCH=$SIZE_MATCH|TAR=$TAR_RC|MANIFEST=$MANIFEST_RC|SUMS=$SUMS_RC\\\" >> \\\"$WORK/logs/critical-results.tsv\\\"\"},{\"line\":94,\"text\":\"# --- Edge-vm chunked lightweight restore/read drill ---\"},{\"line\":108,\"text\":\" rclone copyto \\\"${EDGE_REMOTE}:$EDGE_DEST/MANIFEST.txt\\\" \\\"$WORK/edge/MANIFEST.txt\\\" > \\\"$WORK/logs/edge_manifest_download.out\\\" 2>\\\"$WORK/logs/edge_manifest_download.err\\\"\"},{\"line\":109,\"text\":\" EDGE_MANIFEST_RC=$?\"},{\"line\":111,\"text\":\" rclone copyto \\\"${EDGE_REMOTE}:$EDGE_DEST/SHA256SUMS.txt\\\" \\\"$WORK/edge/SHA256SUMS.txt\\\" > \\\"$WORK/logs/edge_sums_download.out\\\" 2>\\\"$WORK/logs/edge_sums_download.err\\\"\"},{\"line\":114,\"text\":\" rclone copyto \\\"${EDGE_REMOTE}:$EDGE_DEST/$EDGE_ARCHIVE.sha256\\\" \\\"$WORK/edge/$EDGE_ARCHIVE.sha256\\\" > \\\"$WORK/logs/edge_archive_sha_download.out\\\" 2>\\\"$WORK/logs/edge_archive_sha_download.err\\\"\"},{\"line\":117,\"text\":\" rclone lsf \\\"${EDGE_REMOTE}:$EDGE_DEST/parts\\\" --format ps > \\\"$WORK/logs/edge_parts_lsf.txt\\\" 2>\\\"$WORK/logs/edge_parts_lsf.err\\\"\"},{\"line\":130,\"text\":\" rclone copyto \\\"${EDGE_REMOTE}:$EDGE_DEST/parts/$FIRST_PART\\\" \\\"$WORK/edge/$FIRST_PART\\\" > \\\"$WORK/logs/edge_first_part_download.out\\\" 2>\\\"$WORK/logs/edge_first_part_download.err\\\"\"},{\"line\":137,\"text\":\" rclone copyto \\\"${EDGE_REMOTE}:$EDGE_DEST/parts/$LAST_PART\\\" \\\"$WORK/edge/$LAST_PART\\\" > \\\"$WORK/logs/edge_last_part_download.out\\\" 2>\\\"$WORK/logs/edge_last_part_download.err\\\"\"},{\"line\":155,\"text\":\" [ \\\"$EDGE_MANIFEST_RC\\\" = \\\"0\\\" ] || EDGE_STATUS=ERROR\"},{\"line\":172,\"text\":\" NOTES=edge_restore_drill_failed\"},{\"line\":181,\"text\":\" echo \\\"STATUS=OK TS=$TS TYPE=mail-cloud-restore-drill CRITICAL_OK=$CRIT_OK CRITICAL_FAIL=$CRIT_FAIL EDGE_OK=$EDGE_OK EDGE_FAIL=$EDGE_FAIL EDGE_MODE=metadata-first-last-parts SECRET_PRINTED=no NOTES=none\\\" > \\\"$HEALTH\\\"\"},{\"line\":183,\"text\":\" echo \\\"STATUS=ERROR TS=$TS TYPE=mail-cloud-restore-drill CRITICAL_OK=$CRIT_OK CRITICAL_FAIL=$CRIT_FAIL EDGE_OK=$EDGE_OK EDGE_FAIL=$EDGE_FAIL SECRET_PRINTED=no NOTES=$NOTES\\\" > \\\"$HEALTH\\\"\"}],\"path\":\"/usr/local/sbin/homelab-mail-cloud-restore-drill\",\"sha256\":\"c1e13c1813df88095464cf47a989e456c5d8755a425bbb1306908ded6c75cd6b\"},{\"bytes\":12618,\"interface_lines\":[{\"line\":7,\"text\":\"LOCK=/run/lock/homelab-vm-full-restore-drill.lock\"},{\"line\":10,\"text\":\" echo \\\"Another restore drill is already running\\\" >&2\"},{\"line\":16,\"text\":\"LOG=\\\"$LOG_DIR/restore-drill-${RUN_STAMP}.log\\\"\"},{\"line\":17,\"text\":\"HEALTH=\\\"$STATE_DIR/latest-restore-drill.txt\\\"\"},{\"line\":19,\"text\":\"WORK=\\\"$STATE_DIR/restore-drill-${RUN_STAMP}\\\"\"},{\"line\":66,\"text\":\" cp -f \\\"$HEALTH\\\" /var/lib/homelab-health/vm170-vm171-full-restore-drill.txt\"},{\"line\":156,\"text\":\" local actual=\\\"$WORK/vm${source_vmid}-compose-inventory-restored.txt\\\"\"},{\"line\":267,\"text\":\"restore_and_test() {\"},{\"line\":292,\"text\":\" echo \\\"Test VMID $test_vmid already exists; refusing restore\\\" >&2\"},{\"line\":303,\"text\":\" qmrestore \\\\\"},{\"line\":306,\"text\":\" --storage \\\"$RESTORE_STORAGE\\\" \\\\\"},{\"line\":314,\"text\":\" --name \\\"restore-test-${source_vmid}-${RUN_STAMP}\\\"\"},{\"line\":316,\"text\":\" echo \\\"=== RESTORED VM CONFIG ===\\\"\"},{\"line\":322,\"text\":\" echo \\\"Not all restored NICs are link_down=1\\\" >&2\"},{\"line\":336,\"text\":\" echo \\\"=== RESTORE NETWORK ISOLATION PREP ===\\\"\"},{\"line\":337,\"text\":\"RESTORE_TEST_VMID=\\\"${TEST_VMID:-${test_vmid:-}}\\\"\"},{\"line\":338,\"text\":\"if [ -n \\\"$RESTORE_TEST_VMID\\\" ]; then\"},{\"line\":339,\"text\":\" RESTORE_TEST_IP=\\\"$(qm config \\\"$RESTORE_TEST_VMID\\\" | sed -n \\\"s/^ipconfig0: ip=\\\\([^,/]*\\\\).*/\\\\1/p\\\" | head -n1)\\\"\"},{\"line\":340,\"text\":\" echo \\\"RESTORE_TEST_VMID=$RESTORE_TEST_VMID\\\"\"},{\"line\":341,\"text\":\" echo \\\"RESTORE_TEST_IP=$RESTORE_TEST_IP\\\"\"},{\"line\":342,\"text\":\" if [ -n \\\"$RESTORE_TEST_IP\\\" ]; then\"},{\"line\":343,\"text\":\" qm guest exec \\\"$RESTORE_TEST_VMID\\\" --timeout 180 -- bash -lc \\\"ip addr add ${RESTORE_TEST_IP}/32 dev lo 2>/dev/null || true; docker start dockge-agent bookstack-db bookstack prometheus 2>/dev/null || true; sleep 60; docker start dockge-agent bookstack-db bookstack prometheus 2>/dev/null || true; sleep 30\\\" || true\"},{\"line\":348,\"text\":\"# RESTORE DOCKER CHECK OVERRIDE V2 - isolated restore drill semantics\"},{\"line\":351,\"text\":\" local restore_ip=\\\"\\\"\"},{\"line\":352,\"text\":\" restore_ip=\\\"$(qm config \\\"$vmid\\\" | sed -n 's/^ipconfig0: ip=\\\\([^,/]*\\\\).*/\\\\1/p' | head -n1 || true)\\\"\"},{\"line\":354,\"text\":\" echo \\\"RESTORE_TEST_VMID=$vmid\\\"\"},{\"line\":355,\"text\":\" echo \\\"RESTORE_TEST_IP=$restore_ip\\\"\"},{\"line\":357,\"text\":\" if [ -n \\\"$restore_ip\\\" ]; then\"},{\"line\":358,\"text\":\" qm guest exec \\\"$vmid\\\" --timeout 120 -- bash -lc \\\"ip link set lo up || true; ip addr add ${restore_ip}/32 dev lo 2>/dev/null || true\\\" || true\"},{\"line\":379,\"text\":\" echo \\\"RESTORE_DOCKER_CHECK_FAILED_ALLOWED_ONLY_DOCKGE_AGENT\\\"\"},{\"line\":383,\"text\":\" echo \\\"RESTORE_DOCKER_CHECK_OK_ALLOWED_STOPPED=dockge-agent\\\"\"},{\"line\":394,\"text\":\" echo \\\"RESTORE_TEST_VM_${source_vmid}=OK\\\"\"},{\"line\":395,\"text\":\" qm stop \\\"$test_vmid\\\" --skiplock 1 >/dev/null 2>&1 || true; echo \\\"RESTORE_TEST_VM_${source_vmid}_STOPPED_AFTER_CHECK=yes\\\"\"},{\"line\":398,\"text\":\"restore_and_test 170 \\\"$RESTORE_VMID_170\\\"\"},{\"line\":399,\"text\":\"restore_and_test 171 \\\"$RESTORE_VMID_171\\\"\"},{\"line\":411,\"text\":\" echo \\\"TEST_VMIDS=$RESTORE_VMID_170,$RESTORE_VMID_171\\\"\"},{\"line\":424,\"text\":\"cp -f \\\"$HEALTH\\\" /var/lib/homelab-health/vm170-vm171-full-restore-drill.txt\"}],\"path\":\"/usr/local/sbin/homelab-vm-full-restore-drill\",\"sha256\":\"57bba1b7a774b4cfe904dfeb671028bc0427abf04d463e28d8605dbc142faf13\"},{\"bytes\":6851,\"interface_lines\":[{\"line\":4,\"text\":\"CONF=/etc/skladchik-reports-monitor-backup-restore-check.conf\"},{\"line\":9,\"text\":\": \\\"${WORK_ROOT:=/mnt/staging/skladchik-reports-monitor-backup-restore-check}\\\"\"},{\"line\":12,\"text\":\"HEALTH=/var/lib/skladchik-reports-monitor-backup/restore-check-health.txt\"},{\"line\":13,\"text\":\"LOCK=/run/skladchik-reports-monitor-backup-restore-check.lock\"},{\"line\":19,\"text\":\"log(){ printf '%s RESTORE_CHECK %s\\\\n' \\\"$(date -u +%Y-%m-%dT%H:%M:%SZ)\\\" \\\"$*\\\" | tee -a \\\"$LOGDIR/restore-check.log\\\"; }\"},{\"line\":22,\"text\":\"command -v rclone >/dev/null\"},{\"line\":28,\"text\":\" if rclone lsf \\\"${remote_path}.parts/\\\" > \\\"$plist\\\" 2>/dev/null && grep -q '^part-' \\\"$plist\\\"; then\"},{\"line\":29,\"text\":\" while IFS= read -r part; do [ -n \\\"$part\\\" ] && rclone cat \\\"${remote_path}.parts/${part}\\\" >> \\\"$out\\\"; done < <(grep '^part-' \\\"$plist\\\" | sort)\"},{\"line\":31,\"text\":\" rclone copyto \\\"$remote_path\\\" \\\"$out\\\" >/dev/null\"},{\"line\":39,\"text\":\" latest=\\\"$(rclone lsf --dirs-only \\\"${rbase}/\\\" 2>/dev/null | sed 's#/$##' | grep -E '^[0-9]{8}T[0-9]{6}Z$' | sort | tail -n 1)\\\"\"},{\"line\":100,\"text\":\"printf 'STATUS=OK\\\\nTS=%s\\\\nRUN_ID=%s\\\\nREMOTE_BASE=%s\\\\nCLOUD_1_STATUS=%s\\\\nCLOUD_1_RUN=%s\\\\nCLOUD_2_STATUS=%s\\\\nCLOUD_2_RUN=%s\\\\nSHA256_OK=yes\\\\nTAR_OK=yes\\\\nREQUIRED_FILES_OK=yes\\\\nCOOKIE_JAR_OK=yes\\\\nEXTENSION_HEALTH_FILES_OK=yes\\\\nEXTENSION_SCRIPTS_OK=yes\\\\nEXTENSION_UNITS_OK=yes\\\\nSQLITE_OK=yes\\\\nPY_COMPILE_OK=yes\\\\nRESTORE_CHECK_VERSION=4\\\\n' \\\"$(date -u +%Y-%m-%dT%H:%M:%SZ)\\\" \\\"$RUN_ID\\\" \\\"$REMOTE_BASE\\\" \\\"${CLOUD_1_STATUS:-unknown}\\\" \\\"${CLOUD_1_RUN:-}\\\" \\\"${CLOUD_2_STATUS:-unknown}\\\" \\\"${CLOUD_2_RUN:-}\\\" > \\\"$HEALTH\\\"\"}],\"path\":\"/usr/local/sbin/skladchik-reports-monitor-backup-restore-check\",\"sha256\":\"abb2d17392b6edde3603f77e88fc82240c311f932b0d197b1b8289b990c1dc23\"},{\"bytes\":753,\"interface_lines\":[{\"line\":3,\"text\":\"printf 'RESTORE_CHECK_HEALTH_BEGIN\\\\n'\"},{\"line\":4,\"text\":\"cat /var/lib/skladchik-reports-monitor-backup/restore-check-health.txt 2>/dev/null || true\"},{\"line\":5,\"text\":\"printf 'RESTORE_CHECK_HEALTH_END\\\\n'\"},{\"line\":6,\"text\":\"printf 'RESTORE_CHECK_TIMER_BEGIN\\\\n'\"},{\"line\":7,\"text\":\"systemctl is-enabled skladchik-reports-monitor-backup-restore-check.timer 2>/dev/null | sed 's/^/enabled=/' || true\"},{\"line\":8,\"text\":\"systemctl is-active skladchik-reports-monitor-backup-restore-check.timer 2>/dev/null | sed 's/^/active=/' || true\"},{\"line\":9,\"text\":\"systemctl list-timers --all | grep skladchik-reports-monitor-backup-restore-check || true\"},{\"line\":10,\"text\":\"printf 'RESTORE_CHECK_TIMER_END\\\\n'\"},{\"line\":11,\"text\":\"printf 'RESTORE_CHECK_LOG_TAIL_BEGIN\\\\n'\"},{\"line\":12,\"text\":\"tail -n 80 /var/log/skladchik-reports-monitor-backup/restore-check.log 2>/dev/null || true\"},{\"line\":13,\"text\":\"printf 'RESTORE_CHECK_LOG_TAIL_END\\\\n'\"}],\"path\":\"/usr/local/sbin/skladchik-reports-monitor-backup-restore-check-status\",\"sha256\":\"92ca1bd3b9e29bb2adf27deeb009edd8daee70f4fc304a879f79c29e00d99d67\"}]\nRETENTION_SOURCE_SHA256=8adb6c10f5911f39c53775b89eac3cd1823f1ed34b4caccade5279c213e998a3\nRETENTION_CONTRACT={\"lines\":[],\"rc\":65,\"stderr_lines\":[\"GC_STATUS=ERROR|ROOT_CAUSE=GFS_POLICY_HELPER_SHA_DRIFT\"]}\nRETENTION_PLAN={\"delete_count\":0,\"delete_lines\":[],\"keep_count\":0,\"rc\":65,\"stderr_lines\":[\"GC_STATUS=ERROR|ROOT_CAUSE=GFS_POLICY_HELPER_SHA_DRIFT\"],\"stdout_sha256\":\"e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855\",\"summary_lines\":[]}\nALERTMANAGER_READINESS={\"backup_script_exists\":true,\"backup_script_sha256\":\"6deef30b4708a7e5b2ac0f0d98bb6b37b22d858afde994483b67baa18dffdc2a\",\"monitoring_probe\":{\"API_QUERY\":\"OK\",\"AUTH_CONFIGURED\":\"NO\",\"CONFIG_CHECK\":\"OK\",\"EMAIL_CONFIGS\":\"0\",\"RECIPIENT_CONFIGURED\":\"NO\",\"ROUTES\":\"1\",\"SMTP_FROM\":\"NO\",\"SMTP_SMARTHOST\":\"NO\"},\"monitoring_probe_rc\":0,\"monitoring_stderr_lines\":[],\"monitoring_target_present\":true}\nRECIPIENT_SOURCE_COUNTS={\"/etc/aliases\":{\"email_like_count\":0,\"root_alias_present\":false},\"/etc/pve/user.cfg\":{\"email_like_count\":1,\"root_alias_present\":false}}\nISOLATED_PUBLISHER_HINTS={\"lines\":[\"HEAD:docs/operations/workers2/2026-08-23/Workers2_Control_Plane_Remediation_Resume_Plan_20260823.md:3:**Goal:** recover from CMD-1403 without weakening the CR worktree's intentional push protection, publish the durable audit/reference fix through an isolated publisher, install the reference-refresh sanitation fix, and finish with a read-only safe-run gate.\",\"HEAD:docs/operations/workers2/2026-08-23/Workers2_Control_Plane_Remediation_Resume_Plan_20260823.md:17:1. Freshly verify source HEAD/clean state, reference SHA, read/write credentials, remote branch absence, Gitea fetch URL, and `origin` push URL `DISABLED`.\",\"HEAD:docs/operations/workers2/2026-08-23/Workers2_Control_Plane_Remediation_Resume_Plan_20260823.md:28:6. Publish the commit to the explicit Gitea repository URL as the CR branch using the separated write credential and an absent-ref `--force-with-lease`.\",\"HEAD:docs/operations/workers2/2026-08-23/Workers2_Control_Plane_Remediation_Resume_Plan_20260823.md:29:7. Verify exact remote SHA using the explicit repository URL, then delete the temporary publisher clone.\",\"HEAD:docs/operations/workers2/2026-08-23/Workers2_Error_RCA_Ledger_20260823.md:29:- Root cause: v1 reused the permanent `read:repository` credential created for safe-run as a source-write credential. Repository membership had write permission (`mode=2`), but the PAT scope itself was read-only.\",\"HEAD:docs/operations/workers2/2026-08-23/Workers2_Error_RCA_Ledger_20260823.md:32:- Prevention: split read and write credentials; keep the default helper read-only; source push must use a dedicated `write:repository` credential via explicit `HOME`. Source apply must compensate its own local/remote state because installed safe-run does not automatically invoke its rollback hook.\",\"HEAD:docs/operations/workers2/2026-08-23/Workers2_Error_RCA_Ledger_20260823.md:39:- Invalid assumption: a valid fetch URL, valid write token, and successful explicit `ls-remote` do not imply that symbolic remote `origin` uses the same URL for push. Git supports a separate `remote.<name>.pushurl`.\",\"HEAD:docs/operations/workers2/2026-08-23/Workers2_Error_RCA_Ledger_20260823.md:42:- Prevention: `remote.origin.pushurl=DISABLED` is now a required safety invariant. Source publication must never change or bypass that setting from the CR worktree. A temporary isolated publisher clone imports the local commit, publishes through the explicit Gitea URL with the separated write credential and an absent-ref `--force-with-lease`, verifies the exact remote SHA, and is destroyed. All network readback and compensation use the explicit repository URL rather than symbolic `origin`.\",\"HEAD:docs/operations/workers2/2026-08-23/Workers2_GramCrypto_Operational_Audit_20260823.md:67:CMD-1400 proved that the default root Git credential must remain read-only. The v1 remediation correctly installed a `read:repository` credential for `git.gram1.ru`, allowing `homelab-safe-run` to perform its mandatory authenticated `ls-remote` and shallow clone. The same credential cannot be used for `git push`.\",\"HEAD:docs/operations/workers2/2026-08-23/Workers2_GramCrypto_Operational_Audit_20260823.md:74:The write credential is never added to the global Git helper and must never be included in evidence archives.\",\"HEAD:docs/operations/workers2/2026-08-23/Workers2_GramCrypto_Operational_Audit_20260823.md:87:- the target CR ref must be absent before initial publication and creation must use an absent-ref `--force-with-lease`;\",\"HEAD:docs/operations/workers2/2026-08-23/Workers2_Pre_Command_Contract_20260823.md:60:4. the target CR remote ref must be read before push and protected with an exact `--force-with-lease` expectation;\"],\"rc\":0}\nSOURCE_PUBLICATION_STATE={\"cr_branch_sha\":null,\"local_head\":\"50bf24f8ee27f3d2e0d1cbc6c17b62c69ff06bae\",\"ls_remote_rc\":0,\"main_sha\":\"326622a4864fa9b5f43b69981733236bb42c4d7b\",\"pushurl_is_disabled\":true}\nREADINESS_GATES={\"alertmanager_email_route_ready\":false,\"canonical_active_count\":16,\"canonical_coverage_bad\":[\"app-filebrowser\",\"app-hbarhub\",\"app-immich\",\"app-skladchik-reports-monitor\",\"control-bootstrap-v2\",\"vm110\",\"vm111\",\"vm112\",\"vm113\",\"vm130\",\"vm150\",\"vm160\",\"vm170\",\"vm171\",\"vm190\",\"vm9130\"],\"cloud_b_restore_candidate_ready\":false,\"production_audit_rc\":1,\"retention_contract_rc\":65,\"retention_delete_count\":0,\"retention_plan_rc\":65,\"source_offhost_branch_exact\":false}\nNO_PERSISTENT_MUTATION=true\nNO_REPO_WRITE=true\nNO_GIT_FETCH=true\nNO_GIT_COMMIT=true\nNO_GIT_PUSH_UPDATE=true\nNO_SYSTEMD_ACTION=true\nNO_BACKUP_STARTED=true\nNO_RESTORE_STARTED=true\nNO_RETENTION_DELETE=true\nNO_CLOUD_DELETE=true\nNO_MAIL_SENT=true\nDURATION_SEC=9.285\nTASK_RESULT=PASS_HOMELAB_BACKUP_CLOSURE_READINESS_READONLY\nHOMELAB_RESULT_CONTRACT={\"changes_made\":false,\"command_id\":\"SUPPORT-260921-HOMELAB-BACKUP-CLOSURE-READINESS-READONLY-1039R1\",\"rollback_restored\":null,\"rollback_started\":false,\"status\":\"OK\",\"version\":1}\nWORKERS2_BACKUP_CLOSURE_READINESS_END=1\n"
}
View Git Blame Copy Permalink