diff --git a/runtime/history/NEWFI-260923-NEWFDOM-DR221-438-HEAVYLOCK-SCHEDULER-RCA439R1.json b/runtime/history/NEWFI-260923-NEWFDOM-DR221-438-HEAVYLOCK-SCHEDULER-RCA439R1.json new file mode 100644 index 00000000..9d6fc1ba --- /dev/null +++ b/runtime/history/NEWFI-260923-NEWFDOM-DR221-438-HEAVYLOCK-SCHEDULER-RCA439R1.json @@ -0,0 +1,38 @@ +{ + "schema_version": 1, + "channel": "homelab-runtime", + "command_id": "NEWFI-260923-NEWFDOM-DR221-438-HEAVYLOCK-SCHEDULER-RCA439R1", + "status": "OK", + "rc": 0, + "host": "pve01", + "mode": "read-only", + "component": "newfi-dr221-438-heavylock-scheduler-rca", + "started_at_utc": "2026-09-22T22:34:53Z", + "finished_at_utc": "2026-09-22T22:34:53Z", + "reference_register_checked": true, + "reference_sha256": "5a3d8e5154c41cb582a4e0aca68090be1f0138918bf82131a948df326f9d8d66", + "error_register_checked": true, + "error_register_sha256": "3b09a553ec0f527ed3afeed4753f52a74ee3036045bbfb3c685e9f8af4ba7ba0", + "command_sha256": "51f79da5a10e1f18db59f97482f73726947438de73dd916c8f50b0c75f336214", + "duplicate_failed_command_blocked": false, + "block_reason": null, + "execution_started": true, + "change_declared": false, + "result_contract_valid": true, + "result_contract_status": null, + "result_contract_error": null, + "command_rc": 0, + "changes_made": false, + "rollback_started": false, + "rollback_restored": null, + "mutation_outcome": "NO_MUTATION", + "sanitized": true, + "secrets_included": false, + "private_addresses_included": false, + "raw_evidence_retained_locally": true, + "raw_evidence_sha256": "969f0ac21e8ea2afcc013eb098024f91c408e33d61cac081f888e40546c4b569", + "sanitized_output_sha256": "969f0ac21e8ea2afcc013eb098024f91c408e33d61cac081f888e40546c4b569", + "output_truncated_in_json": false, + "full_sanitized_output_url": "https://git.gram1.ru/.well-known/homelab-runtime/latest.txt", + "output": "NEWFI439_BEGIN=true\nCOMMAND_ID=NEWFI-260923-NEWFDOM-DR221-438-HEAVYLOCK-SCHEDULER-RCA439R1\nMODE=read-only\nCOMPONENT=newfi-dr221-438-heavylock-scheduler-rca\nPREDECESSOR=NEWFI-260922-NEWFDOM-DR221-FINAL-ISOLATED-RESTORE438R1\nTARGET_FAILURE=GLOBAL_HEAVY_LOCK_WAIT_TIMEOUT_3600S\nMUTATION_BOUNDARY=NONE;SYSTEMD_TOPOLOGY_AND_OWNER_PROGRESS_READ_ONLY;NO_TIMER_ACTION;NO_SERVICE_ACTION;NO_SIGNAL;NO_LOCK_BREAK;NO_BACKUP_START;NO_VM_WRITE;NO_CLOUD_WRITE\nREFERENCE_SHA256=5a3d8e5154c41cb582a4e0aca68090be1f0138918bf82131a948df326f9d8d66\nERROR_REGISTER_SHA256=3b09a553ec0f527ed3afeed4753f52a74ee3036045bbfb3c685e9f8af4ba7ba0\nRUNNER_SHA256=b248a4c32c9cc64e5747e7dce6c7fc0a23f5124a77c71ce72e27a81aceae9d2d\nFRESHNESS_SHA256=2a9b391e05fd38755f18d2c920ded3758cf162fdb332ab84203475f9017d3558\nAUDIT_SHA256=f7b936fa997bdefdd0c3c5d1d96e496888ebebc8ea85a60ed3bf5bcc9bf4962a\nWRAPPER_SHA256=5077444065c732451cb84898680f62361b21a24ef9eb4f191253e82ead524ea2\nVM210_FRESHNESS_RC=0\nFRESHNESS_STATUS=OK\nVMID=210\nDECISION=SEALED_PROVEN\nGENERATION=vm210-pve01-20260922T172531Z\nUPDATED_AT_UTC=2026-09-22T17:27:10Z\nSEALED_EPOCH=1790098030\nSTATE_FILE=/var/lib/homelab-backup/cloud-quorum/vm210/vm210-pve01-20260922T172531Z.json\nRESTORE_EVIDENCE=/var/lib/homelab-backup/vm-restore-validation/vm210/vm210-pve01-20260922T172531Z.json\nVM210_SEALED_PROVEN_CURRENT=true\nSYSTEMD_TOPOLOGY_BEGIN=true\nType=oneshot\nRestart=no\nMainPID=1488093\nResult=success\nExecMainStartTimestamp=Tue 2026-09-22 20:41:06 MSK\nExecMainExitTimestamp=\nExecMainStatus=0\nTriggers=\nTriggeredBy=homelab-backup-v2.timer\nLoadState=loaded\nActiveState=activating\nSubState=start\nUnitFileState=static\nCanStop=yes\nRefuseManualStop=no\nInvocationID=567686e8c1b44972aa666eb7991b69d4\nUnit=homelab-backup-v2.service\nNextElapseUSecRealtime=\nLastTriggerUSec=Tue 2026-09-22 20:41:06 MSK\nAccuracyUSec=1min\nRandomizedDelayUSec=10min\nPersistent=no\nTriggers=homelab-backup-v2.service\nLoadState=loaded\nActiveState=active\nSubState=running\nUnitFileState=enabled\nCanStop=yes\nRefuseManualStop=no\nSERVICE_UNIT_SAFE_LINES_BEGIN=true\n# /etc/systemd/system/homelab-backup-v2.service\n[Unit]\nDescription=Homelab Backup V2 manifest-driven scheduler\nAfter=network-online.target\nWants=network-online.target\n\n[Service]\nType=oneshot\nUser=root\nGroup=root\nUMask=0077\nExecStart=/usr/local/sbin/homelab-backup-v2 scheduled\nNice=10\nIOSchedulingClass=best-effort\nIOSchedulingPriority=7\nTimeoutStartSec=8h\nSERVICE_UNIT_SAFE_LINES_END=true\nTIMER_UNIT_SAFE_LINES_BEGIN=true\n# /etc/systemd/system/homelab-backup-v2.timer\n[Unit]\nDescription=Homelab Backup V2 scheduler timer\n\n[Timer]\nOnActiveSec=20min\nOnUnitActiveSec=30min\nRandomizedDelaySec=10min\nUnit=homelab-backup-v2.service\n\n[Install]\nWantedBy=timers.target\nTIMER_UNIT_SAFE_LINES_END=true\nREVERSE_TRIGGER_UNITS_BEGIN=true\nhomelab-backup-v2.service\nREVERSE_TRIGGER_UNITS_END=true\nTIMER_ENABLED=enabled\nTIMER_ACTIVE=active\nSERVICE_UNITFILE_STATE=static\nSERVICE_RESTART=no\nSERVICE_TYPE=oneshot\nSERVICE_TRIGGERED_BY=homelab-backup-v2.timer\nTIMER_TRIGGERS=homelab-backup-v2.service\nTIMER_ONLY_MAINTENANCE_WINDOW_TOPOLOGY=true\nSYSTEMD_TOPOLOGY_END=true\nOWNER_SNAPSHOT_BEGIN=A\nOWNER_STATE_A|service=activating|sub=start|pid=1488093|invocation=567686e8c1b44972aa666eb7991b69d4|heavy=BUSY\n1488093 1 17594 S 0.0 0.0 homelab-backup- /usr/bin/python3 /usr/local/sbin/homelab-backup-v2 scheduled\nOWNER_WCHAN_A=poll_schedule_timeout.constprop.0\nOWNER_IO_A_rchar=3714373731\nOWNER_IO_A_wchar=2345094021\nOWNER_IO_A_syscr=1449408\nOWNER_IO_A_syscw=166852\nOWNER_IO_A_read_bytes=1454080\nOWNER_IO_A_write_bytes=986578944\nOWNER_PROCESS_TREE_A\n 1273 1 2192560 S 0.0 0.0 sshd sshd: /usr/sbin/sshd -D [listener] 0 of 10-100 startups\n 3566 3346 2192542 S 0.0 0.0 sshd sshd: /usr/sbin/sshd -D [listener] 0 of 10-100 startups\n 4026 3805 2192539 S 0.0 0.0 sshd sshd: /usr/sbin/sshd -D [listener] 0 of 10-100 startups\n1488093 1 17594 S 0.0 0.0 homelab-backup- /usr/bin/python3 /usr/local/sbin/homelab-backup-v2 scheduled\n2468623 1488093 1 S 0.5 0.0 ssh /usr/bin/ssh -o BatchMode=yes -o StrictHostKeyChecking=yes root@pve02 PATH=/usr/sbin:/usr/bin:/sbin:/bin; export PATH; exec qm guest exec 160 --synchronous 1 --timeout 60 -- /usr/bin/python3 -I -B -c 'import base64,sys; p=sys.argv[1]; o=int(sys.argv[2]); n=int(sys.argv[3]); f=open(p,'\"'\"'rb'\"'\"'); f.seek(o); b=f.read(n); f.close(); sys.stdout.write(base64.b64encode(b).decode())' /tmp/homelab-v2-xf-0922T174106Ze0d1cc1923e5.tar.zst 986578944 131072\n2468663 1273 0 S 0.0 0.0 sshd-session sshd-session: root [priv]\n2468694 2468663 0 S 0.0 0.0 sshd-session sshd-session: root@notty\n2468806 2468805 0 S 0.0 0.0 sed sed -E -e s#(https?://)[^/@[:space:]]+@#\\1[REDACTED]@#g -e s#(ssh://)[^/@[:space:]]+@#\\1[REDACTED]@#g -e s#(git@[REDACTED]:[:space:]]+#\\1[REDACTED]#g -e s/([0-9]{1,3}\\.){3}[0-9]{1,3}/[PRIVATE_IP]/g -e s/[A-Za-z0-9._%+-]+@[A-Za-z0-9.-]+\\.[A-Za-z]{2,}/[EMAIL]/g\nHEAVY_LOCK_HOLDERS_A\nCURRENT_INVOCATION_JOURNAL_A_LINES=1\nCURRENT_INVOCATION_JOURNAL_A_SHA256=99fbe57d7379e6a73b514b4c18ab500149de436d3635e4d1b604f7f9800d93bb\nCURRENT_INVOCATION_MARKERS_A\n2026-09-22T20:41:06+03:00 pve01 homelab-backup-v2[1488093]: SELFTEST_REPLACEMENT486=PASS\nOWNER_SNAPSHOT_END=A\nOWNER_SNAPSHOT_BEGIN=B\nOWNER_STATE_B|service=activating|sub=start|pid=1488093|invocation=567686e8c1b44972aa666eb7991b69d4|heavy=BUSY\n1488093 1 17624 S 0.0 0.0 homelab-backup- /usr/bin/python3 /usr/local/sbin/homelab-backup-v2 scheduled\nOWNER_WCHAN_B=poll_schedule_timeout.constprop.0\nOWNER_IO_B_rchar=3720682590\nOWNER_IO_B_wchar=2349144257\nOWNER_IO_B_syscr=1451910\nOWNER_IO_B_syscw=167139\nOWNER_IO_B_read_bytes=1454080\nOWNER_IO_B_write_bytes=988282880\nOWNER_PROCESS_TREE_B\n 1273 1 2192590 S 0.0 0.0 sshd sshd: /usr/sbin/sshd -D [listener] 0 of 10-100 startups\n 3566 3346 2192572 S 0.0 0.0 sshd sshd: /usr/sbin/sshd -D [listener] 0 of 10-100 startups\n 4026 3805 2192569 S 0.0 0.0 sshd sshd: /usr/sbin/sshd -D [listener] 0 of 10-100 startups\n1488093 1 17624 S 0.0 0.0 homelab-backup- /usr/bin/python3 /usr/local/sbin/homelab-backup-v2 scheduled\n2468663 1273 30 S 0.0 0.0 sshd-session sshd-session: root [priv]\n2468694 2468663 30 S 0.0 0.0 sshd-session sshd-session: root@notty\n2469027 1488093 1 S 0.5 0.0 ssh /usr/bin/ssh -o BatchMode=yes -o StrictHostKeyChecking=yes root@pve02 PATH=/usr/sbin:/usr/bin:/sbin:/bin; export PATH; exec qm guest exec 160 --synchronous 1 --timeout 60 -- /usr/bin/python3 -I -B -c 'import base64,sys; p=sys.argv[1]; o=int(sys.argv[2]); n=int(sys.argv[3]); f=open(p,'\"'\"'rb'\"'\"'); f.seek(o); b=f.read(n); f.close(); sys.stdout.write(base64.b64encode(b).decode())' /tmp/homelab-v2-xf-0922T174106Ze0d1cc1923e5.tar.zst 988282880 131072\n2469118 2469117 0 S 0.0 0.0 sed sed -E -e s#(https?://)[^/@[:space:]]+@#\\1[REDACTED]@#g -e s#(ssh://)[^/@[:space:]]+@#\\1[REDACTED]@#g -e s#(git@[REDACTED]:[:space:]]+#\\1[REDACTED]#g -e s/([0-9]{1,3}\\.){3}[0-9]{1,3}/[PRIVATE_IP]/g -e s/[A-Za-z0-9._%+-]+@[A-Za-z0-9.-]+\\.[A-Za-z]{2,}/[EMAIL]/g\nHEAVY_LOCK_HOLDERS_B\nCURRENT_INVOCATION_JOURNAL_B_LINES=1\nCURRENT_INVOCATION_JOURNAL_B_SHA256=99fbe57d7379e6a73b514b4c18ab500149de436d3635e4d1b604f7f9800d93bb\nCURRENT_INVOCATION_MARKERS_B\n2026-09-22T20:41:06+03:00 pve01 homelab-backup-v2[1488093]: SELFTEST_REPLACEMENT486=PASS\nOWNER_SNAPSHOT_END=B\nOWNER_DELTA_rchar=6308859\nOWNER_DELTA_wchar=4050236\nOWNER_DELTA_syscr=2502\nOWNER_DELTA_syscw=287\nOWNER_DELTA_read_bytes=0\nOWNER_DELTA_write_bytes=1703936\nOWNER_IO_PROGRESS_30S=true\nOWNER_FINAL|service=activating|sub=start|pid=1488093|heavy=BUSY\nAUDIT_RC=1\nISSUE|severity=HIGH|code=RPO_BREACH|logical_id=vm112|root_cause=NO_FRESH_V2_OR_SEALED_QUORUM\nISSUE|severity=HIGH|code=RPO_BREACH|logical_id=vm113|root_cause=NO_FRESH_V2_OR_SEALED_QUORUM\nISSUE|severity=HIGH|code=RPO_BREACH|logical_id=vm130|root_cause=NO_FRESH_V2_OR_SEALED_QUORUM\nISSUE|severity=HIGH|code=RPO_BREACH|logical_id=vm150|root_cause=NO_FRESH_V2_OR_SEALED_QUORUM\nISSUE|severity=HIGH|code=RPO_BREACH|logical_id=vm160|root_cause=NO_FRESH_V2_OR_SEALED_QUORUM\nISSUE|severity=HIGH|code=RPO_BREACH|logical_id=vm170|root_cause=NO_FRESH_V2_OR_SEALED_QUORUM\nISSUE|severity=HIGH|code=RPO_BREACH|logical_id=vm171|root_cause=NO_FRESH_V2_OR_SEALED_QUORUM\nISSUE|severity=HIGH|code=RPO_BREACH|logical_id=vm190|root_cause=NO_FRESH_V2_OR_SEALED_QUORUM\nISSUE|severity=HIGH|code=RPO_BREACH|logical_id=vm9130|root_cause=NO_FRESH_V2_OR_SEALED_QUORUM\nISSUE|severity=HIGH|code=LOCK_BUSY|logical_id=/run/homelab-backup-heavy.lock|root_cause=ACTIVE_OR_ORPHAN_LOCK\nSUMMARY|AUDIT_STATUS=OK|BACKUP_STATUS=RED|ACTIVE_COUNT=16|ISSUE_COUNT=10|CLOUD_A=OK|CLOUD_B=OK|CLOUD_B_DRILL=OK|DR_PROVEN=YES|SOURCE_OFFHOST=OK\nAUDIT_NEWFDOM_ISSUE_COUNT=0\nAUDIT_UNEXPECTED_CODE_COUNT=0\nLOCK_RCA_CLASS=LEGITIMATE_LONG_OWNER_WITH_TIMER_ONLY_RETRIGGER_TOPOLOGY\nSAFE_MAINTENANCE_COORDINATION=STOP_TIMER_ONLY_WAIT_CURRENT_SERVICE_NATURAL_EXIT_ACQUIRE_HEAVY_LOCK_RESTORE_TIMER_ON_ALL_PATHS\nDECISION=PASS_DR221_439_SAFE_TIMER_MAINTENANCE_WINDOW_PROVEN\nNEXT_GATE=FINAL_DR221_RESTORE_WITH_TEMPORARY_TIMER_PAUSE_NATURAL_OWNER_DRAIN_AND_GUARANTEED_TIMER_RESTORE\nCHANGES_MADE_BY_439=false\nPRODUCT_MUTATION_BY_439=false\nVM_MUTATION_BY_439=false\nCLOUD_MUTATION_BY_439=false\nSYSTEMD_MUTATION_BY_439=false\nHOMELAB_RESULT_CONTRACT={\"version\":1,\"command_id\":\"NEWFI-260923-NEWFDOM-DR221-438-HEAVYLOCK-SCHEDULER-RCA439R1\",\"status\":\"OK\",\"changes_made\":false,\"rollback_started\":false,\"rollback_restored\":null}\nNEWFI439_END=true\n" +} diff --git a/runtime/history/NEWFI-260923-NEWFDOM-DR221-438-HEAVYLOCK-SCHEDULER-RCA439R1.txt b/runtime/history/NEWFI-260923-NEWFDOM-DR221-438-HEAVYLOCK-SCHEDULER-RCA439R1.txt new file mode 100644 index 00000000..becd6b59 --- /dev/null +++ b/runtime/history/NEWFI-260923-NEWFDOM-DR221-438-HEAVYLOCK-SCHEDULER-RCA439R1.txt @@ -0,0 +1,212 @@ +CHAT_OUTPUT_BEGIN +COMMAND_ID=NEWFI-260923-NEWFDOM-DR221-438-HEAVYLOCK-SCHEDULER-RCA439R1 +STATUS=OK +RC=0 +HOST=pve01 +MODE=read-only +COMPONENT=newfi-dr221-438-heavylock-scheduler-rca +REFERENCE_REGISTER_CHECK=OK +REFERENCE_SHA256=5a3d8e5154c41cb582a4e0aca68090be1f0138918bf82131a948df326f9d8d66 +ERROR_REGISTER_CHECK=OK +ERROR_REGISTER_SHA256=3b09a553ec0f527ed3afeed4753f52a74ee3036045bbfb3c685e9f8af4ba7ba0 +COMMAND_SHA256=51f79da5a10e1f18db59f97482f73726947438de73dd916c8f50b0c75f336214 +DUPLICATE_FAILED_COMMAND_BLOCKED=false +EXECUTION_STARTED=true +CHANGE_DECLARED=false +RESULT_CONTRACT_VALID=true +RESULT_CONTRACT_STATUS=NOT_APPLICABLE +RESULT_CONTRACT_ERROR=NONE +COMMAND_RC=0 +CHANGES_MADE=false +ROLLBACK_STARTED=false +ROLLBACK_RESTORED=null +MUTATION_OUTCOME=NO_MUTATION +SANITIZED=yes +SECRETS_INCLUDED=no +PRIVATE_ADDRESSES_INCLUDED=no +RAW_EVIDENCE_SHA256=969f0ac21e8ea2afcc013eb098024f91c408e33d61cac081f888e40546c4b569 +SANITIZED_OUTPUT_SHA256=969f0ac21e8ea2afcc013eb098024f91c408e33d61cac081f888e40546c4b569 +OUTPUT_BEGIN +NEWFI439_BEGIN=true +COMMAND_ID=NEWFI-260923-NEWFDOM-DR221-438-HEAVYLOCK-SCHEDULER-RCA439R1 +MODE=read-only +COMPONENT=newfi-dr221-438-heavylock-scheduler-rca +PREDECESSOR=NEWFI-260922-NEWFDOM-DR221-FINAL-ISOLATED-RESTORE438R1 +TARGET_FAILURE=GLOBAL_HEAVY_LOCK_WAIT_TIMEOUT_3600S +MUTATION_BOUNDARY=NONE;SYSTEMD_TOPOLOGY_AND_OWNER_PROGRESS_READ_ONLY;NO_TIMER_ACTION;NO_SERVICE_ACTION;NO_SIGNAL;NO_LOCK_BREAK;NO_BACKUP_START;NO_VM_WRITE;NO_CLOUD_WRITE +REFERENCE_SHA256=5a3d8e5154c41cb582a4e0aca68090be1f0138918bf82131a948df326f9d8d66 +ERROR_REGISTER_SHA256=3b09a553ec0f527ed3afeed4753f52a74ee3036045bbfb3c685e9f8af4ba7ba0 +RUNNER_SHA256=b248a4c32c9cc64e5747e7dce6c7fc0a23f5124a77c71ce72e27a81aceae9d2d +FRESHNESS_SHA256=2a9b391e05fd38755f18d2c920ded3758cf162fdb332ab84203475f9017d3558 +AUDIT_SHA256=f7b936fa997bdefdd0c3c5d1d96e496888ebebc8ea85a60ed3bf5bcc9bf4962a +WRAPPER_SHA256=5077444065c732451cb84898680f62361b21a24ef9eb4f191253e82ead524ea2 +VM210_FRESHNESS_RC=0 +FRESHNESS_STATUS=OK +VMID=210 +DECISION=SEALED_PROVEN +GENERATION=vm210-pve01-20260922T172531Z +UPDATED_AT_UTC=2026-09-22T17:27:10Z +SEALED_EPOCH=1790098030 +STATE_FILE=/var/lib/homelab-backup/cloud-quorum/vm210/vm210-pve01-20260922T172531Z.json +RESTORE_EVIDENCE=/var/lib/homelab-backup/vm-restore-validation/vm210/vm210-pve01-20260922T172531Z.json +VM210_SEALED_PROVEN_CURRENT=true +SYSTEMD_TOPOLOGY_BEGIN=true +Type=oneshot +Restart=no +MainPID=1488093 +Result=success +ExecMainStartTimestamp=Tue 2026-09-22 20:41:06 MSK +ExecMainExitTimestamp= +ExecMainStatus=0 +Triggers= +TriggeredBy=homelab-backup-v2.timer +LoadState=loaded +ActiveState=activating +SubState=start +UnitFileState=static +CanStop=yes +RefuseManualStop=no +InvocationID=567686e8c1b44972aa666eb7991b69d4 +Unit=homelab-backup-v2.service +NextElapseUSecRealtime= +LastTriggerUSec=Tue 2026-09-22 20:41:06 MSK +AccuracyUSec=1min +RandomizedDelayUSec=10min +Persistent=no +Triggers=homelab-backup-v2.service +LoadState=loaded +ActiveState=active +SubState=running +UnitFileState=enabled +CanStop=yes +RefuseManualStop=no +SERVICE_UNIT_SAFE_LINES_BEGIN=true +# /etc/systemd/system/homelab-backup-v2.service +[Unit] +Description=Homelab Backup V2 manifest-driven scheduler +After=network-online.target +Wants=network-online.target + +[Service] +Type=oneshot +User=root +Group=root +UMask=0077 +ExecStart=/usr/local/sbin/homelab-backup-v2 scheduled +Nice=10 +IOSchedulingClass=best-effort +IOSchedulingPriority=7 +TimeoutStartSec=8h +SERVICE_UNIT_SAFE_LINES_END=true +TIMER_UNIT_SAFE_LINES_BEGIN=true +# /etc/systemd/system/homelab-backup-v2.timer +[Unit] +Description=Homelab Backup V2 scheduler timer + +[Timer] +OnActiveSec=20min +OnUnitActiveSec=30min +RandomizedDelaySec=10min +Unit=homelab-backup-v2.service + +[Install] +WantedBy=timers.target +TIMER_UNIT_SAFE_LINES_END=true +REVERSE_TRIGGER_UNITS_BEGIN=true +homelab-backup-v2.service +REVERSE_TRIGGER_UNITS_END=true +TIMER_ENABLED=enabled +TIMER_ACTIVE=active +SERVICE_UNITFILE_STATE=static +SERVICE_RESTART=no +SERVICE_TYPE=oneshot +SERVICE_TRIGGERED_BY=homelab-backup-v2.timer +TIMER_TRIGGERS=homelab-backup-v2.service +TIMER_ONLY_MAINTENANCE_WINDOW_TOPOLOGY=true +SYSTEMD_TOPOLOGY_END=true +OWNER_SNAPSHOT_BEGIN=A +OWNER_STATE_A|service=activating|sub=start|pid=1488093|invocation=567686e8c1b44972aa666eb7991b69d4|heavy=BUSY +1488093 1 17594 S 0.0 0.0 homelab-backup- /usr/bin/python3 /usr/local/sbin/homelab-backup-v2 scheduled +OWNER_WCHAN_A=poll_schedule_timeout.constprop.0 +OWNER_IO_A_rchar=3714373731 +OWNER_IO_A_wchar=2345094021 +OWNER_IO_A_syscr=1449408 +OWNER_IO_A_syscw=166852 +OWNER_IO_A_read_bytes=1454080 +OWNER_IO_A_write_bytes=986578944 +OWNER_PROCESS_TREE_A + 1273 1 2192560 S 0.0 0.0 sshd sshd: /usr/sbin/sshd -D [listener] 0 of 10-100 startups + 3566 3346 2192542 S 0.0 0.0 sshd sshd: /usr/sbin/sshd -D [listener] 0 of 10-100 startups + 4026 3805 2192539 S 0.0 0.0 sshd sshd: /usr/sbin/sshd -D [listener] 0 of 10-100 startups +1488093 1 17594 S 0.0 0.0 homelab-backup- /usr/bin/python3 /usr/local/sbin/homelab-backup-v2 scheduled +2468623 1488093 1 S 0.5 0.0 ssh /usr/bin/ssh -o BatchMode=yes -o StrictHostKeyChecking=yes root@pve02 PATH=/usr/sbin:/usr/bin:/sbin:/bin; export PATH; exec qm guest exec 160 --synchronous 1 --timeout 60 -- /usr/bin/python3 -I -B -c 'import base64,sys; p=sys.argv[1]; o=int(sys.argv[2]); n=int(sys.argv[3]); f=open(p,'"'"'rb'"'"'); f.seek(o); b=f.read(n); f.close(); sys.stdout.write(base64.b64encode(b).decode())' /tmp/homelab-v2-xf-0922T174106Ze0d1cc1923e5.tar.zst 986578944 131072 +2468663 1273 0 S 0.0 0.0 sshd-session sshd-session: root [priv] +2468694 2468663 0 S 0.0 0.0 sshd-session sshd-session: root@notty +2468806 2468805 0 S 0.0 0.0 sed sed -E -e s#(https?://)[^/@[:space:]]+@#\1[REDACTED]@#g -e s#(ssh://)[^/@[:space:]]+@#\1[REDACTED]@#g -e s#(git@[REDACTED]:[:space:]]+#\1[REDACTED]#g -e s/([0-9]{1,3}\.){3}[0-9]{1,3}/[PRIVATE_IP]/g -e s/[A-Za-z0-9._%+-]+@[A-Za-z0-9.-]+\.[A-Za-z]{2,}/[EMAIL]/g +HEAVY_LOCK_HOLDERS_A +CURRENT_INVOCATION_JOURNAL_A_LINES=1 +CURRENT_INVOCATION_JOURNAL_A_SHA256=99fbe57d7379e6a73b514b4c18ab500149de436d3635e4d1b604f7f9800d93bb +CURRENT_INVOCATION_MARKERS_A +2026-09-22T20:41:06+03:00 pve01 homelab-backup-v2[1488093]: SELFTEST_REPLACEMENT486=PASS +OWNER_SNAPSHOT_END=A +OWNER_SNAPSHOT_BEGIN=B +OWNER_STATE_B|service=activating|sub=start|pid=1488093|invocation=567686e8c1b44972aa666eb7991b69d4|heavy=BUSY +1488093 1 17624 S 0.0 0.0 homelab-backup- /usr/bin/python3 /usr/local/sbin/homelab-backup-v2 scheduled +OWNER_WCHAN_B=poll_schedule_timeout.constprop.0 +OWNER_IO_B_rchar=3720682590 +OWNER_IO_B_wchar=2349144257 +OWNER_IO_B_syscr=1451910 +OWNER_IO_B_syscw=167139 +OWNER_IO_B_read_bytes=1454080 +OWNER_IO_B_write_bytes=988282880 +OWNER_PROCESS_TREE_B + 1273 1 2192590 S 0.0 0.0 sshd sshd: /usr/sbin/sshd -D [listener] 0 of 10-100 startups + 3566 3346 2192572 S 0.0 0.0 sshd sshd: /usr/sbin/sshd -D [listener] 0 of 10-100 startups + 4026 3805 2192569 S 0.0 0.0 sshd sshd: /usr/sbin/sshd -D [listener] 0 of 10-100 startups +1488093 1 17624 S 0.0 0.0 homelab-backup- /usr/bin/python3 /usr/local/sbin/homelab-backup-v2 scheduled +2468663 1273 30 S 0.0 0.0 sshd-session sshd-session: root [priv] +2468694 2468663 30 S 0.0 0.0 sshd-session sshd-session: root@notty +2469027 1488093 1 S 0.5 0.0 ssh /usr/bin/ssh -o BatchMode=yes -o StrictHostKeyChecking=yes root@pve02 PATH=/usr/sbin:/usr/bin:/sbin:/bin; export PATH; exec qm guest exec 160 --synchronous 1 --timeout 60 -- /usr/bin/python3 -I -B -c 'import base64,sys; p=sys.argv[1]; o=int(sys.argv[2]); n=int(sys.argv[3]); f=open(p,'"'"'rb'"'"'); f.seek(o); b=f.read(n); f.close(); sys.stdout.write(base64.b64encode(b).decode())' /tmp/homelab-v2-xf-0922T174106Ze0d1cc1923e5.tar.zst 988282880 131072 +2469118 2469117 0 S 0.0 0.0 sed sed -E -e s#(https?://)[^/@[:space:]]+@#\1[REDACTED]@#g -e s#(ssh://)[^/@[:space:]]+@#\1[REDACTED]@#g -e s#(git@[REDACTED]:[:space:]]+#\1[REDACTED]#g -e s/([0-9]{1,3}\.){3}[0-9]{1,3}/[PRIVATE_IP]/g -e s/[A-Za-z0-9._%+-]+@[A-Za-z0-9.-]+\.[A-Za-z]{2,}/[EMAIL]/g +HEAVY_LOCK_HOLDERS_B +CURRENT_INVOCATION_JOURNAL_B_LINES=1 +CURRENT_INVOCATION_JOURNAL_B_SHA256=99fbe57d7379e6a73b514b4c18ab500149de436d3635e4d1b604f7f9800d93bb +CURRENT_INVOCATION_MARKERS_B +2026-09-22T20:41:06+03:00 pve01 homelab-backup-v2[1488093]: SELFTEST_REPLACEMENT486=PASS +OWNER_SNAPSHOT_END=B +OWNER_DELTA_rchar=6308859 +OWNER_DELTA_wchar=4050236 +OWNER_DELTA_syscr=2502 +OWNER_DELTA_syscw=287 +OWNER_DELTA_read_bytes=0 +OWNER_DELTA_write_bytes=1703936 +OWNER_IO_PROGRESS_30S=true +OWNER_FINAL|service=activating|sub=start|pid=1488093|heavy=BUSY +AUDIT_RC=1 +ISSUE|severity=HIGH|code=RPO_BREACH|logical_id=vm112|root_cause=NO_FRESH_V2_OR_SEALED_QUORUM +ISSUE|severity=HIGH|code=RPO_BREACH|logical_id=vm113|root_cause=NO_FRESH_V2_OR_SEALED_QUORUM +ISSUE|severity=HIGH|code=RPO_BREACH|logical_id=vm130|root_cause=NO_FRESH_V2_OR_SEALED_QUORUM +ISSUE|severity=HIGH|code=RPO_BREACH|logical_id=vm150|root_cause=NO_FRESH_V2_OR_SEALED_QUORUM +ISSUE|severity=HIGH|code=RPO_BREACH|logical_id=vm160|root_cause=NO_FRESH_V2_OR_SEALED_QUORUM +ISSUE|severity=HIGH|code=RPO_BREACH|logical_id=vm170|root_cause=NO_FRESH_V2_OR_SEALED_QUORUM +ISSUE|severity=HIGH|code=RPO_BREACH|logical_id=vm171|root_cause=NO_FRESH_V2_OR_SEALED_QUORUM +ISSUE|severity=HIGH|code=RPO_BREACH|logical_id=vm190|root_cause=NO_FRESH_V2_OR_SEALED_QUORUM +ISSUE|severity=HIGH|code=RPO_BREACH|logical_id=vm9130|root_cause=NO_FRESH_V2_OR_SEALED_QUORUM +ISSUE|severity=HIGH|code=LOCK_BUSY|logical_id=/run/homelab-backup-heavy.lock|root_cause=ACTIVE_OR_ORPHAN_LOCK +SUMMARY|AUDIT_STATUS=OK|BACKUP_STATUS=RED|ACTIVE_COUNT=16|ISSUE_COUNT=10|CLOUD_A=OK|CLOUD_B=OK|CLOUD_B_DRILL=OK|DR_PROVEN=YES|SOURCE_OFFHOST=OK +AUDIT_NEWFDOM_ISSUE_COUNT=0 +AUDIT_UNEXPECTED_CODE_COUNT=0 +LOCK_RCA_CLASS=LEGITIMATE_LONG_OWNER_WITH_TIMER_ONLY_RETRIGGER_TOPOLOGY +SAFE_MAINTENANCE_COORDINATION=STOP_TIMER_ONLY_WAIT_CURRENT_SERVICE_NATURAL_EXIT_ACQUIRE_HEAVY_LOCK_RESTORE_TIMER_ON_ALL_PATHS +DECISION=PASS_DR221_439_SAFE_TIMER_MAINTENANCE_WINDOW_PROVEN +NEXT_GATE=FINAL_DR221_RESTORE_WITH_TEMPORARY_TIMER_PAUSE_NATURAL_OWNER_DRAIN_AND_GUARANTEED_TIMER_RESTORE +CHANGES_MADE_BY_439=false +PRODUCT_MUTATION_BY_439=false +VM_MUTATION_BY_439=false +CLOUD_MUTATION_BY_439=false +SYSTEMD_MUTATION_BY_439=false +HOMELAB_RESULT_CONTRACT={"version":1,"command_id":"NEWFI-260923-NEWFDOM-DR221-438-HEAVYLOCK-SCHEDULER-RCA439R1","status":"OK","changes_made":false,"rollback_started":false,"rollback_restored":null} +NEWFI439_END=true + +OUTPUT_END +CHAT_OUTPUT_END diff --git a/runtime/latest.json b/runtime/latest.json index fa610f27..9d6fc1ba 100644 --- a/runtime/latest.json +++ b/runtime/latest.json @@ -1,38 +1,38 @@ { "schema_version": 1, "channel": "homelab-runtime", - "command_id": "NEWFI-260922-NEWFDOM-DR221-FINAL-ISOLATED-RESTORE438R1", - "status": "FAIL", - "rc": 5, + "command_id": "NEWFI-260923-NEWFDOM-DR221-438-HEAVYLOCK-SCHEDULER-RCA439R1", + "status": "OK", + "rc": 0, "host": "pve01", - "mode": "change", - "component": "newfi-dr221-final-isolated-restore-acceptance-437", - "started_at_utc": "2026-09-22T22:25:55Z", - "finished_at_utc": "2026-09-22T22:25:55Z", + "mode": "read-only", + "component": "newfi-dr221-438-heavylock-scheduler-rca", + "started_at_utc": "2026-09-22T22:34:53Z", + "finished_at_utc": "2026-09-22T22:34:53Z", "reference_register_checked": true, "reference_sha256": "5a3d8e5154c41cb582a4e0aca68090be1f0138918bf82131a948df326f9d8d66", "error_register_checked": true, "error_register_sha256": "3b09a553ec0f527ed3afeed4753f52a74ee3036045bbfb3c685e9f8af4ba7ba0", - "command_sha256": "c40ca75d93f5487b08adf55b1efc96d566b0164435923b2bb84485ed2862a342", + "command_sha256": "51f79da5a10e1f18db59f97482f73726947438de73dd916c8f50b0c75f336214", "duplicate_failed_command_blocked": false, "block_reason": null, "execution_started": true, - "change_declared": true, + "change_declared": false, "result_contract_valid": true, - "result_contract_status": "FAIL", + "result_contract_status": null, "result_contract_error": null, - "command_rc": 5, - "changes_made": true, - "rollback_started": true, - "rollback_restored": true, - "mutation_outcome": "ROLLED_BACK", + "command_rc": 0, + "changes_made": false, + "rollback_started": false, + "rollback_restored": null, + "mutation_outcome": "NO_MUTATION", "sanitized": true, "secrets_included": false, "private_addresses_included": false, "raw_evidence_retained_locally": true, - "raw_evidence_sha256": "79e238f968290832f27370590912a9923c609c80dd3c71f046ccaa7e70c053de", - "sanitized_output_sha256": "79e238f968290832f27370590912a9923c609c80dd3c71f046ccaa7e70c053de", + "raw_evidence_sha256": "969f0ac21e8ea2afcc013eb098024f91c408e33d61cac081f888e40546c4b569", + "sanitized_output_sha256": "969f0ac21e8ea2afcc013eb098024f91c408e33d61cac081f888e40546c4b569", "output_truncated_in_json": false, "full_sanitized_output_url": "https://git.gram1.ru/.well-known/homelab-runtime/latest.txt", - "output": "NEWFI438_BEGIN=true\nCOMMAND_ID=NEWFI-260922-NEWFDOM-DR221-FINAL-ISOLATED-RESTORE438R1\nMODE=change\nCOMPONENT=newfi-dr221-final-isolated-restore-acceptance-437\nANCHOR_425_FIRST_COMMIT=595fac75a29e35f975cda1a51fab6919fce528d8\nPREDECESSOR_437=PASS_DR221_437_FINAL_SOURCE_PRODUCT_CONTRACT_PROVEN\nDR221_BACKUP_GENERATION=vm210-pve01-20260922T172531Z\nTEST_VMID=9210\nAUDIT_POLICY=CORE_HEALTH_PLUS_NO_NEW_UNEXPECTED_OR_NEWFDOM_ISSUES;RPO_BACKLOG_EXTERNAL\nMUTATION_BOUNDARY=CLOUD_A_READ_ONLY_RECONSTRUCTION;TEMP_VM_RESTORE_9210_ONLY;TEMP_VM_NIC_ISOLATION;TEMP_BOOT_QGA_VERIFY;TEMP_DESTROY;LOCAL_FULL_RESTORE_EVIDENCE;NO_PRODUCTION_VM_CONFIG_WRITE;NO_CLOUD_WRITE_OR_DELETE\nREFERENCE_SHA256=5a3d8e5154c41cb582a4e0aca68090be1f0138918bf82131a948df326f9d8d66\nERROR_REGISTER_SHA256=3b09a553ec0f527ed3afeed4753f52a74ee3036045bbfb3c685e9f8af4ba7ba0\nRUNNER_SHA256=b248a4c32c9cc64e5747e7dce6c7fc0a23f5124a77c71ce72e27a81aceae9d2d\nQUEUE_SHA256=96280069bbe3c4930105f8f00e49a299a24da299eb84f2c0a43360afa4c9713e\nFRESHNESS_SHA256=2a9b391e05fd38755f18d2c920ded3758cf162fdb332ab84203475f9017d3558\nWORKER_SHA256=4c729debe80f99528fea1f6e1723e7c4d03c5249ba01fd640aaa7c4a4075075c\nAUDIT_SHA256=f7b936fa997bdefdd0c3c5d1d96e496888ebebc8ea85a60ed3bf5bcc9bf4962a\nWORKER_PLAN_RC=0\nPLAN_STATUS=OK\nVMID=210\nOWNER=pve01\nTYPE=qemu\nEXPECTED_STATUS=running\nFRESHNESS_RC=0\nFRESHNESS_STATUS=OK\nVMID=210\nDECISION=SEALED_PROVEN\nGENERATION=vm210-pve01-20260922T172531Z\nUPDATED_AT_UTC=2026-09-22T17:27:10Z\nSEALED_EPOCH=1790098030\nSTATE_FILE=/var/lib/homelab-backup/cloud-quorum/vm210/vm210-pve01-20260922T172531Z.json\nRESTORE_EVIDENCE=/var/lib/homelab-backup/vm-restore-validation/vm210/vm210-pve01-20260922T172531Z.json\nSOURCE_AGENT_PING_RC=0\nSOURCE_PRODUCT_TRANSPORT_RC=0\nSOURCE_PRODUCT_GUEST|exited=1|exitcode=0\nSOURCE_PRODUCT_SIGNATURE_SHA256=fcb934facb4ce56d534ba49759b29a4f7c59a39848a779186ba325ecbad9c02a\nAUDIT_ENTRY_RC=1\nISSUE|severity=HIGH|code=RPO_BREACH|logical_id=vm112|root_cause=NO_FRESH_V2_OR_SEALED_QUORUM\nISSUE|severity=HIGH|code=RPO_BREACH|logical_id=vm113|root_cause=NO_FRESH_V2_OR_SEALED_QUORUM\nISSUE|severity=HIGH|code=RPO_BREACH|logical_id=vm130|root_cause=NO_FRESH_V2_OR_SEALED_QUORUM\nISSUE|severity=HIGH|code=RPO_BREACH|logical_id=vm150|root_cause=NO_FRESH_V2_OR_SEALED_QUORUM\nISSUE|severity=HIGH|code=RPO_BREACH|logical_id=vm160|root_cause=NO_FRESH_V2_OR_SEALED_QUORUM\nISSUE|severity=HIGH|code=RPO_BREACH|logical_id=vm170|root_cause=NO_FRESH_V2_OR_SEALED_QUORUM\nISSUE|severity=HIGH|code=RPO_BREACH|logical_id=vm171|root_cause=NO_FRESH_V2_OR_SEALED_QUORUM\nISSUE|severity=HIGH|code=RPO_BREACH|logical_id=vm190|root_cause=NO_FRESH_V2_OR_SEALED_QUORUM\nISSUE|severity=HIGH|code=LOCK_BUSY|logical_id=/run/homelab-backup-heavy.lock|root_cause=ACTIVE_OR_ORPHAN_LOCK\nSUMMARY|AUDIT_STATUS=OK|BACKUP_STATUS=RED|ACTIVE_COUNT=16|ISSUE_COUNT=9|CLOUD_A=OK|CLOUD_B=OK|CLOUD_B_DRILL=OK|DR_PROVEN=YES|SOURCE_OFFHOST=OK\nGLOBAL_HEAVY_LOCK_WAIT_BEGIN=true\nGLOBAL_HEAVY_LOCK_WAIT_SECONDS=3600\nGLOBAL_HEAVY_LOCK_ACQUIRE_RC=1\nFAILURE=GLOBAL_HEAVY_LOCK_WAIT_TIMEOUT\nTEMP_CLEANUP_ROLLBACK_RESTORED=true\nTEMP_CLEANUP_LV_RESIDUE_COUNT=0\nCLEANUP_ACCEPTANCE=ROLLBACK_PROVEN\nDECISION=HOLD_DR221_438_FINAL_RESTORE_INCOMPLETE\nDR221_CLOSED=false\nNEWFDOM_PROJECT_COMPLETE=false\nFULL_PRODUCT_REMAINING=1\nNEXT_GATE=RECONCILE_ONLY_DR221_438_FAILURE\nTASK_RESULT=HOLD_NEWFDOM_DR221_FINAL_RESTORE\nCHANGES_MADE_BY_438=true\nPRODUCT_MUTATION_BY_438=false\nPRODUCTION_VM_CONFIG_MUTATION_BY_438=false\nCLOUD_MUTATION_BY_438=false\nTEMP_VM_MUTATION_BY_438=false\nROLLBACK_STARTED=true\nROLLBACK_RESTORED=true\nHOMELAB_RESULT_CONTRACT={\"version\":1,\"command_id\":\"NEWFI-260922-NEWFDOM-DR221-FINAL-ISOLATED-RESTORE438R1\",\"status\":\"FAIL\",\"changes_made\":true,\"rollback_started\":true,\"rollback_restored\":true}\nNEWFI438_END=true\n" + "output": "NEWFI439_BEGIN=true\nCOMMAND_ID=NEWFI-260923-NEWFDOM-DR221-438-HEAVYLOCK-SCHEDULER-RCA439R1\nMODE=read-only\nCOMPONENT=newfi-dr221-438-heavylock-scheduler-rca\nPREDECESSOR=NEWFI-260922-NEWFDOM-DR221-FINAL-ISOLATED-RESTORE438R1\nTARGET_FAILURE=GLOBAL_HEAVY_LOCK_WAIT_TIMEOUT_3600S\nMUTATION_BOUNDARY=NONE;SYSTEMD_TOPOLOGY_AND_OWNER_PROGRESS_READ_ONLY;NO_TIMER_ACTION;NO_SERVICE_ACTION;NO_SIGNAL;NO_LOCK_BREAK;NO_BACKUP_START;NO_VM_WRITE;NO_CLOUD_WRITE\nREFERENCE_SHA256=5a3d8e5154c41cb582a4e0aca68090be1f0138918bf82131a948df326f9d8d66\nERROR_REGISTER_SHA256=3b09a553ec0f527ed3afeed4753f52a74ee3036045bbfb3c685e9f8af4ba7ba0\nRUNNER_SHA256=b248a4c32c9cc64e5747e7dce6c7fc0a23f5124a77c71ce72e27a81aceae9d2d\nFRESHNESS_SHA256=2a9b391e05fd38755f18d2c920ded3758cf162fdb332ab84203475f9017d3558\nAUDIT_SHA256=f7b936fa997bdefdd0c3c5d1d96e496888ebebc8ea85a60ed3bf5bcc9bf4962a\nWRAPPER_SHA256=5077444065c732451cb84898680f62361b21a24ef9eb4f191253e82ead524ea2\nVM210_FRESHNESS_RC=0\nFRESHNESS_STATUS=OK\nVMID=210\nDECISION=SEALED_PROVEN\nGENERATION=vm210-pve01-20260922T172531Z\nUPDATED_AT_UTC=2026-09-22T17:27:10Z\nSEALED_EPOCH=1790098030\nSTATE_FILE=/var/lib/homelab-backup/cloud-quorum/vm210/vm210-pve01-20260922T172531Z.json\nRESTORE_EVIDENCE=/var/lib/homelab-backup/vm-restore-validation/vm210/vm210-pve01-20260922T172531Z.json\nVM210_SEALED_PROVEN_CURRENT=true\nSYSTEMD_TOPOLOGY_BEGIN=true\nType=oneshot\nRestart=no\nMainPID=1488093\nResult=success\nExecMainStartTimestamp=Tue 2026-09-22 20:41:06 MSK\nExecMainExitTimestamp=\nExecMainStatus=0\nTriggers=\nTriggeredBy=homelab-backup-v2.timer\nLoadState=loaded\nActiveState=activating\nSubState=start\nUnitFileState=static\nCanStop=yes\nRefuseManualStop=no\nInvocationID=567686e8c1b44972aa666eb7991b69d4\nUnit=homelab-backup-v2.service\nNextElapseUSecRealtime=\nLastTriggerUSec=Tue 2026-09-22 20:41:06 MSK\nAccuracyUSec=1min\nRandomizedDelayUSec=10min\nPersistent=no\nTriggers=homelab-backup-v2.service\nLoadState=loaded\nActiveState=active\nSubState=running\nUnitFileState=enabled\nCanStop=yes\nRefuseManualStop=no\nSERVICE_UNIT_SAFE_LINES_BEGIN=true\n# /etc/systemd/system/homelab-backup-v2.service\n[Unit]\nDescription=Homelab Backup V2 manifest-driven scheduler\nAfter=network-online.target\nWants=network-online.target\n\n[Service]\nType=oneshot\nUser=root\nGroup=root\nUMask=0077\nExecStart=/usr/local/sbin/homelab-backup-v2 scheduled\nNice=10\nIOSchedulingClass=best-effort\nIOSchedulingPriority=7\nTimeoutStartSec=8h\nSERVICE_UNIT_SAFE_LINES_END=true\nTIMER_UNIT_SAFE_LINES_BEGIN=true\n# /etc/systemd/system/homelab-backup-v2.timer\n[Unit]\nDescription=Homelab Backup V2 scheduler timer\n\n[Timer]\nOnActiveSec=20min\nOnUnitActiveSec=30min\nRandomizedDelaySec=10min\nUnit=homelab-backup-v2.service\n\n[Install]\nWantedBy=timers.target\nTIMER_UNIT_SAFE_LINES_END=true\nREVERSE_TRIGGER_UNITS_BEGIN=true\nhomelab-backup-v2.service\nREVERSE_TRIGGER_UNITS_END=true\nTIMER_ENABLED=enabled\nTIMER_ACTIVE=active\nSERVICE_UNITFILE_STATE=static\nSERVICE_RESTART=no\nSERVICE_TYPE=oneshot\nSERVICE_TRIGGERED_BY=homelab-backup-v2.timer\nTIMER_TRIGGERS=homelab-backup-v2.service\nTIMER_ONLY_MAINTENANCE_WINDOW_TOPOLOGY=true\nSYSTEMD_TOPOLOGY_END=true\nOWNER_SNAPSHOT_BEGIN=A\nOWNER_STATE_A|service=activating|sub=start|pid=1488093|invocation=567686e8c1b44972aa666eb7991b69d4|heavy=BUSY\n1488093 1 17594 S 0.0 0.0 homelab-backup- /usr/bin/python3 /usr/local/sbin/homelab-backup-v2 scheduled\nOWNER_WCHAN_A=poll_schedule_timeout.constprop.0\nOWNER_IO_A_rchar=3714373731\nOWNER_IO_A_wchar=2345094021\nOWNER_IO_A_syscr=1449408\nOWNER_IO_A_syscw=166852\nOWNER_IO_A_read_bytes=1454080\nOWNER_IO_A_write_bytes=986578944\nOWNER_PROCESS_TREE_A\n 1273 1 2192560 S 0.0 0.0 sshd sshd: /usr/sbin/sshd -D [listener] 0 of 10-100 startups\n 3566 3346 2192542 S 0.0 0.0 sshd sshd: /usr/sbin/sshd -D [listener] 0 of 10-100 startups\n 4026 3805 2192539 S 0.0 0.0 sshd sshd: /usr/sbin/sshd -D [listener] 0 of 10-100 startups\n1488093 1 17594 S 0.0 0.0 homelab-backup- /usr/bin/python3 /usr/local/sbin/homelab-backup-v2 scheduled\n2468623 1488093 1 S 0.5 0.0 ssh /usr/bin/ssh -o BatchMode=yes -o StrictHostKeyChecking=yes root@pve02 PATH=/usr/sbin:/usr/bin:/sbin:/bin; export PATH; exec qm guest exec 160 --synchronous 1 --timeout 60 -- /usr/bin/python3 -I -B -c 'import base64,sys; p=sys.argv[1]; o=int(sys.argv[2]); n=int(sys.argv[3]); f=open(p,'\"'\"'rb'\"'\"'); f.seek(o); b=f.read(n); f.close(); sys.stdout.write(base64.b64encode(b).decode())' /tmp/homelab-v2-xf-0922T174106Ze0d1cc1923e5.tar.zst 986578944 131072\n2468663 1273 0 S 0.0 0.0 sshd-session sshd-session: root [priv]\n2468694 2468663 0 S 0.0 0.0 sshd-session sshd-session: root@notty\n2468806 2468805 0 S 0.0 0.0 sed sed -E -e s#(https?://)[^/@[:space:]]+@#\\1[REDACTED]@#g -e s#(ssh://)[^/@[:space:]]+@#\\1[REDACTED]@#g -e s#(git@[REDACTED]:[:space:]]+#\\1[REDACTED]#g -e s/([0-9]{1,3}\\.){3}[0-9]{1,3}/[PRIVATE_IP]/g -e s/[A-Za-z0-9._%+-]+@[A-Za-z0-9.-]+\\.[A-Za-z]{2,}/[EMAIL]/g\nHEAVY_LOCK_HOLDERS_A\nCURRENT_INVOCATION_JOURNAL_A_LINES=1\nCURRENT_INVOCATION_JOURNAL_A_SHA256=99fbe57d7379e6a73b514b4c18ab500149de436d3635e4d1b604f7f9800d93bb\nCURRENT_INVOCATION_MARKERS_A\n2026-09-22T20:41:06+03:00 pve01 homelab-backup-v2[1488093]: SELFTEST_REPLACEMENT486=PASS\nOWNER_SNAPSHOT_END=A\nOWNER_SNAPSHOT_BEGIN=B\nOWNER_STATE_B|service=activating|sub=start|pid=1488093|invocation=567686e8c1b44972aa666eb7991b69d4|heavy=BUSY\n1488093 1 17624 S 0.0 0.0 homelab-backup- /usr/bin/python3 /usr/local/sbin/homelab-backup-v2 scheduled\nOWNER_WCHAN_B=poll_schedule_timeout.constprop.0\nOWNER_IO_B_rchar=3720682590\nOWNER_IO_B_wchar=2349144257\nOWNER_IO_B_syscr=1451910\nOWNER_IO_B_syscw=167139\nOWNER_IO_B_read_bytes=1454080\nOWNER_IO_B_write_bytes=988282880\nOWNER_PROCESS_TREE_B\n 1273 1 2192590 S 0.0 0.0 sshd sshd: /usr/sbin/sshd -D [listener] 0 of 10-100 startups\n 3566 3346 2192572 S 0.0 0.0 sshd sshd: /usr/sbin/sshd -D [listener] 0 of 10-100 startups\n 4026 3805 2192569 S 0.0 0.0 sshd sshd: /usr/sbin/sshd -D [listener] 0 of 10-100 startups\n1488093 1 17624 S 0.0 0.0 homelab-backup- /usr/bin/python3 /usr/local/sbin/homelab-backup-v2 scheduled\n2468663 1273 30 S 0.0 0.0 sshd-session sshd-session: root [priv]\n2468694 2468663 30 S 0.0 0.0 sshd-session sshd-session: root@notty\n2469027 1488093 1 S 0.5 0.0 ssh /usr/bin/ssh -o BatchMode=yes -o StrictHostKeyChecking=yes root@pve02 PATH=/usr/sbin:/usr/bin:/sbin:/bin; export PATH; exec qm guest exec 160 --synchronous 1 --timeout 60 -- /usr/bin/python3 -I -B -c 'import base64,sys; p=sys.argv[1]; o=int(sys.argv[2]); n=int(sys.argv[3]); f=open(p,'\"'\"'rb'\"'\"'); f.seek(o); b=f.read(n); f.close(); sys.stdout.write(base64.b64encode(b).decode())' /tmp/homelab-v2-xf-0922T174106Ze0d1cc1923e5.tar.zst 988282880 131072\n2469118 2469117 0 S 0.0 0.0 sed sed -E -e s#(https?://)[^/@[:space:]]+@#\\1[REDACTED]@#g -e s#(ssh://)[^/@[:space:]]+@#\\1[REDACTED]@#g -e s#(git@[REDACTED]:[:space:]]+#\\1[REDACTED]#g -e s/([0-9]{1,3}\\.){3}[0-9]{1,3}/[PRIVATE_IP]/g -e s/[A-Za-z0-9._%+-]+@[A-Za-z0-9.-]+\\.[A-Za-z]{2,}/[EMAIL]/g\nHEAVY_LOCK_HOLDERS_B\nCURRENT_INVOCATION_JOURNAL_B_LINES=1\nCURRENT_INVOCATION_JOURNAL_B_SHA256=99fbe57d7379e6a73b514b4c18ab500149de436d3635e4d1b604f7f9800d93bb\nCURRENT_INVOCATION_MARKERS_B\n2026-09-22T20:41:06+03:00 pve01 homelab-backup-v2[1488093]: SELFTEST_REPLACEMENT486=PASS\nOWNER_SNAPSHOT_END=B\nOWNER_DELTA_rchar=6308859\nOWNER_DELTA_wchar=4050236\nOWNER_DELTA_syscr=2502\nOWNER_DELTA_syscw=287\nOWNER_DELTA_read_bytes=0\nOWNER_DELTA_write_bytes=1703936\nOWNER_IO_PROGRESS_30S=true\nOWNER_FINAL|service=activating|sub=start|pid=1488093|heavy=BUSY\nAUDIT_RC=1\nISSUE|severity=HIGH|code=RPO_BREACH|logical_id=vm112|root_cause=NO_FRESH_V2_OR_SEALED_QUORUM\nISSUE|severity=HIGH|code=RPO_BREACH|logical_id=vm113|root_cause=NO_FRESH_V2_OR_SEALED_QUORUM\nISSUE|severity=HIGH|code=RPO_BREACH|logical_id=vm130|root_cause=NO_FRESH_V2_OR_SEALED_QUORUM\nISSUE|severity=HIGH|code=RPO_BREACH|logical_id=vm150|root_cause=NO_FRESH_V2_OR_SEALED_QUORUM\nISSUE|severity=HIGH|code=RPO_BREACH|logical_id=vm160|root_cause=NO_FRESH_V2_OR_SEALED_QUORUM\nISSUE|severity=HIGH|code=RPO_BREACH|logical_id=vm170|root_cause=NO_FRESH_V2_OR_SEALED_QUORUM\nISSUE|severity=HIGH|code=RPO_BREACH|logical_id=vm171|root_cause=NO_FRESH_V2_OR_SEALED_QUORUM\nISSUE|severity=HIGH|code=RPO_BREACH|logical_id=vm190|root_cause=NO_FRESH_V2_OR_SEALED_QUORUM\nISSUE|severity=HIGH|code=RPO_BREACH|logical_id=vm9130|root_cause=NO_FRESH_V2_OR_SEALED_QUORUM\nISSUE|severity=HIGH|code=LOCK_BUSY|logical_id=/run/homelab-backup-heavy.lock|root_cause=ACTIVE_OR_ORPHAN_LOCK\nSUMMARY|AUDIT_STATUS=OK|BACKUP_STATUS=RED|ACTIVE_COUNT=16|ISSUE_COUNT=10|CLOUD_A=OK|CLOUD_B=OK|CLOUD_B_DRILL=OK|DR_PROVEN=YES|SOURCE_OFFHOST=OK\nAUDIT_NEWFDOM_ISSUE_COUNT=0\nAUDIT_UNEXPECTED_CODE_COUNT=0\nLOCK_RCA_CLASS=LEGITIMATE_LONG_OWNER_WITH_TIMER_ONLY_RETRIGGER_TOPOLOGY\nSAFE_MAINTENANCE_COORDINATION=STOP_TIMER_ONLY_WAIT_CURRENT_SERVICE_NATURAL_EXIT_ACQUIRE_HEAVY_LOCK_RESTORE_TIMER_ON_ALL_PATHS\nDECISION=PASS_DR221_439_SAFE_TIMER_MAINTENANCE_WINDOW_PROVEN\nNEXT_GATE=FINAL_DR221_RESTORE_WITH_TEMPORARY_TIMER_PAUSE_NATURAL_OWNER_DRAIN_AND_GUARANTEED_TIMER_RESTORE\nCHANGES_MADE_BY_439=false\nPRODUCT_MUTATION_BY_439=false\nVM_MUTATION_BY_439=false\nCLOUD_MUTATION_BY_439=false\nSYSTEMD_MUTATION_BY_439=false\nHOMELAB_RESULT_CONTRACT={\"version\":1,\"command_id\":\"NEWFI-260923-NEWFDOM-DR221-438-HEAVYLOCK-SCHEDULER-RCA439R1\",\"status\":\"OK\",\"changes_made\":false,\"rollback_started\":false,\"rollback_restored\":null}\nNEWFI439_END=true\n" } diff --git a/runtime/latest.txt b/runtime/latest.txt index 9c2f1bcd..becd6b59 100644 --- a/runtime/latest.txt +++ b/runtime/latest.txt @@ -1,56 +1,46 @@ CHAT_OUTPUT_BEGIN -COMMAND_ID=NEWFI-260922-NEWFDOM-DR221-FINAL-ISOLATED-RESTORE438R1 -STATUS=FAIL -RC=5 +COMMAND_ID=NEWFI-260923-NEWFDOM-DR221-438-HEAVYLOCK-SCHEDULER-RCA439R1 +STATUS=OK +RC=0 HOST=pve01 -MODE=change -COMPONENT=newfi-dr221-final-isolated-restore-acceptance-437 +MODE=read-only +COMPONENT=newfi-dr221-438-heavylock-scheduler-rca REFERENCE_REGISTER_CHECK=OK REFERENCE_SHA256=5a3d8e5154c41cb582a4e0aca68090be1f0138918bf82131a948df326f9d8d66 ERROR_REGISTER_CHECK=OK ERROR_REGISTER_SHA256=3b09a553ec0f527ed3afeed4753f52a74ee3036045bbfb3c685e9f8af4ba7ba0 -COMMAND_SHA256=c40ca75d93f5487b08adf55b1efc96d566b0164435923b2bb84485ed2862a342 +COMMAND_SHA256=51f79da5a10e1f18db59f97482f73726947438de73dd916c8f50b0c75f336214 DUPLICATE_FAILED_COMMAND_BLOCKED=false EXECUTION_STARTED=true -CHANGE_DECLARED=true +CHANGE_DECLARED=false RESULT_CONTRACT_VALID=true -RESULT_CONTRACT_STATUS=FAIL +RESULT_CONTRACT_STATUS=NOT_APPLICABLE RESULT_CONTRACT_ERROR=NONE -COMMAND_RC=5 -CHANGES_MADE=true -ROLLBACK_STARTED=true -ROLLBACK_RESTORED=true -MUTATION_OUTCOME=ROLLED_BACK +COMMAND_RC=0 +CHANGES_MADE=false +ROLLBACK_STARTED=false +ROLLBACK_RESTORED=null +MUTATION_OUTCOME=NO_MUTATION SANITIZED=yes SECRETS_INCLUDED=no PRIVATE_ADDRESSES_INCLUDED=no -RAW_EVIDENCE_SHA256=79e238f968290832f27370590912a9923c609c80dd3c71f046ccaa7e70c053de -SANITIZED_OUTPUT_SHA256=79e238f968290832f27370590912a9923c609c80dd3c71f046ccaa7e70c053de +RAW_EVIDENCE_SHA256=969f0ac21e8ea2afcc013eb098024f91c408e33d61cac081f888e40546c4b569 +SANITIZED_OUTPUT_SHA256=969f0ac21e8ea2afcc013eb098024f91c408e33d61cac081f888e40546c4b569 OUTPUT_BEGIN -NEWFI438_BEGIN=true -COMMAND_ID=NEWFI-260922-NEWFDOM-DR221-FINAL-ISOLATED-RESTORE438R1 -MODE=change -COMPONENT=newfi-dr221-final-isolated-restore-acceptance-437 -ANCHOR_425_FIRST_COMMIT=595fac75a29e35f975cda1a51fab6919fce528d8 -PREDECESSOR_437=PASS_DR221_437_FINAL_SOURCE_PRODUCT_CONTRACT_PROVEN -DR221_BACKUP_GENERATION=vm210-pve01-20260922T172531Z -TEST_VMID=9210 -AUDIT_POLICY=CORE_HEALTH_PLUS_NO_NEW_UNEXPECTED_OR_NEWFDOM_ISSUES;RPO_BACKLOG_EXTERNAL -MUTATION_BOUNDARY=CLOUD_A_READ_ONLY_RECONSTRUCTION;TEMP_VM_RESTORE_9210_ONLY;TEMP_VM_NIC_ISOLATION;TEMP_BOOT_QGA_VERIFY;TEMP_DESTROY;LOCAL_FULL_RESTORE_EVIDENCE;NO_PRODUCTION_VM_CONFIG_WRITE;NO_CLOUD_WRITE_OR_DELETE +NEWFI439_BEGIN=true +COMMAND_ID=NEWFI-260923-NEWFDOM-DR221-438-HEAVYLOCK-SCHEDULER-RCA439R1 +MODE=read-only +COMPONENT=newfi-dr221-438-heavylock-scheduler-rca +PREDECESSOR=NEWFI-260922-NEWFDOM-DR221-FINAL-ISOLATED-RESTORE438R1 +TARGET_FAILURE=GLOBAL_HEAVY_LOCK_WAIT_TIMEOUT_3600S +MUTATION_BOUNDARY=NONE;SYSTEMD_TOPOLOGY_AND_OWNER_PROGRESS_READ_ONLY;NO_TIMER_ACTION;NO_SERVICE_ACTION;NO_SIGNAL;NO_LOCK_BREAK;NO_BACKUP_START;NO_VM_WRITE;NO_CLOUD_WRITE REFERENCE_SHA256=5a3d8e5154c41cb582a4e0aca68090be1f0138918bf82131a948df326f9d8d66 ERROR_REGISTER_SHA256=3b09a553ec0f527ed3afeed4753f52a74ee3036045bbfb3c685e9f8af4ba7ba0 RUNNER_SHA256=b248a4c32c9cc64e5747e7dce6c7fc0a23f5124a77c71ce72e27a81aceae9d2d -QUEUE_SHA256=96280069bbe3c4930105f8f00e49a299a24da299eb84f2c0a43360afa4c9713e FRESHNESS_SHA256=2a9b391e05fd38755f18d2c920ded3758cf162fdb332ab84203475f9017d3558 -WORKER_SHA256=4c729debe80f99528fea1f6e1723e7c4d03c5249ba01fd640aaa7c4a4075075c AUDIT_SHA256=f7b936fa997bdefdd0c3c5d1d96e496888ebebc8ea85a60ed3bf5bcc9bf4962a -WORKER_PLAN_RC=0 -PLAN_STATUS=OK -VMID=210 -OWNER=pve01 -TYPE=qemu -EXPECTED_STATUS=running -FRESHNESS_RC=0 +WRAPPER_SHA256=5077444065c732451cb84898680f62361b21a24ef9eb4f191253e82ead524ea2 +VM210_FRESHNESS_RC=0 FRESHNESS_STATUS=OK VMID=210 DECISION=SEALED_PROVEN @@ -59,11 +49,140 @@ UPDATED_AT_UTC=2026-09-22T17:27:10Z SEALED_EPOCH=1790098030 STATE_FILE=/var/lib/homelab-backup/cloud-quorum/vm210/vm210-pve01-20260922T172531Z.json RESTORE_EVIDENCE=/var/lib/homelab-backup/vm-restore-validation/vm210/vm210-pve01-20260922T172531Z.json -SOURCE_AGENT_PING_RC=0 -SOURCE_PRODUCT_TRANSPORT_RC=0 -SOURCE_PRODUCT_GUEST|exited=1|exitcode=0 -SOURCE_PRODUCT_SIGNATURE_SHA256=fcb934facb4ce56d534ba49759b29a4f7c59a39848a779186ba325ecbad9c02a -AUDIT_ENTRY_RC=1 +VM210_SEALED_PROVEN_CURRENT=true +SYSTEMD_TOPOLOGY_BEGIN=true +Type=oneshot +Restart=no +MainPID=1488093 +Result=success +ExecMainStartTimestamp=Tue 2026-09-22 20:41:06 MSK +ExecMainExitTimestamp= +ExecMainStatus=0 +Triggers= +TriggeredBy=homelab-backup-v2.timer +LoadState=loaded +ActiveState=activating +SubState=start +UnitFileState=static +CanStop=yes +RefuseManualStop=no +InvocationID=567686e8c1b44972aa666eb7991b69d4 +Unit=homelab-backup-v2.service +NextElapseUSecRealtime= +LastTriggerUSec=Tue 2026-09-22 20:41:06 MSK +AccuracyUSec=1min +RandomizedDelayUSec=10min +Persistent=no +Triggers=homelab-backup-v2.service +LoadState=loaded +ActiveState=active +SubState=running +UnitFileState=enabled +CanStop=yes +RefuseManualStop=no +SERVICE_UNIT_SAFE_LINES_BEGIN=true +# /etc/systemd/system/homelab-backup-v2.service +[Unit] +Description=Homelab Backup V2 manifest-driven scheduler +After=network-online.target +Wants=network-online.target + +[Service] +Type=oneshot +User=root +Group=root +UMask=0077 +ExecStart=/usr/local/sbin/homelab-backup-v2 scheduled +Nice=10 +IOSchedulingClass=best-effort +IOSchedulingPriority=7 +TimeoutStartSec=8h +SERVICE_UNIT_SAFE_LINES_END=true +TIMER_UNIT_SAFE_LINES_BEGIN=true +# /etc/systemd/system/homelab-backup-v2.timer +[Unit] +Description=Homelab Backup V2 scheduler timer + +[Timer] +OnActiveSec=20min +OnUnitActiveSec=30min +RandomizedDelaySec=10min +Unit=homelab-backup-v2.service + +[Install] +WantedBy=timers.target +TIMER_UNIT_SAFE_LINES_END=true +REVERSE_TRIGGER_UNITS_BEGIN=true +homelab-backup-v2.service +REVERSE_TRIGGER_UNITS_END=true +TIMER_ENABLED=enabled +TIMER_ACTIVE=active +SERVICE_UNITFILE_STATE=static +SERVICE_RESTART=no +SERVICE_TYPE=oneshot +SERVICE_TRIGGERED_BY=homelab-backup-v2.timer +TIMER_TRIGGERS=homelab-backup-v2.service +TIMER_ONLY_MAINTENANCE_WINDOW_TOPOLOGY=true +SYSTEMD_TOPOLOGY_END=true +OWNER_SNAPSHOT_BEGIN=A +OWNER_STATE_A|service=activating|sub=start|pid=1488093|invocation=567686e8c1b44972aa666eb7991b69d4|heavy=BUSY +1488093 1 17594 S 0.0 0.0 homelab-backup- /usr/bin/python3 /usr/local/sbin/homelab-backup-v2 scheduled +OWNER_WCHAN_A=poll_schedule_timeout.constprop.0 +OWNER_IO_A_rchar=3714373731 +OWNER_IO_A_wchar=2345094021 +OWNER_IO_A_syscr=1449408 +OWNER_IO_A_syscw=166852 +OWNER_IO_A_read_bytes=1454080 +OWNER_IO_A_write_bytes=986578944 +OWNER_PROCESS_TREE_A + 1273 1 2192560 S 0.0 0.0 sshd sshd: /usr/sbin/sshd -D [listener] 0 of 10-100 startups + 3566 3346 2192542 S 0.0 0.0 sshd sshd: /usr/sbin/sshd -D [listener] 0 of 10-100 startups + 4026 3805 2192539 S 0.0 0.0 sshd sshd: /usr/sbin/sshd -D [listener] 0 of 10-100 startups +1488093 1 17594 S 0.0 0.0 homelab-backup- /usr/bin/python3 /usr/local/sbin/homelab-backup-v2 scheduled +2468623 1488093 1 S 0.5 0.0 ssh /usr/bin/ssh -o BatchMode=yes -o StrictHostKeyChecking=yes root@pve02 PATH=/usr/sbin:/usr/bin:/sbin:/bin; export PATH; exec qm guest exec 160 --synchronous 1 --timeout 60 -- /usr/bin/python3 -I -B -c 'import base64,sys; p=sys.argv[1]; o=int(sys.argv[2]); n=int(sys.argv[3]); f=open(p,'"'"'rb'"'"'); f.seek(o); b=f.read(n); f.close(); sys.stdout.write(base64.b64encode(b).decode())' /tmp/homelab-v2-xf-0922T174106Ze0d1cc1923e5.tar.zst 986578944 131072 +2468663 1273 0 S 0.0 0.0 sshd-session sshd-session: root [priv] +2468694 2468663 0 S 0.0 0.0 sshd-session sshd-session: root@notty +2468806 2468805 0 S 0.0 0.0 sed sed -E -e s#(https?://)[^/@[:space:]]+@#\1[REDACTED]@#g -e s#(ssh://)[^/@[:space:]]+@#\1[REDACTED]@#g -e s#(git@[REDACTED]:[:space:]]+#\1[REDACTED]#g -e s/([0-9]{1,3}\.){3}[0-9]{1,3}/[PRIVATE_IP]/g -e s/[A-Za-z0-9._%+-]+@[A-Za-z0-9.-]+\.[A-Za-z]{2,}/[EMAIL]/g +HEAVY_LOCK_HOLDERS_A +CURRENT_INVOCATION_JOURNAL_A_LINES=1 +CURRENT_INVOCATION_JOURNAL_A_SHA256=99fbe57d7379e6a73b514b4c18ab500149de436d3635e4d1b604f7f9800d93bb +CURRENT_INVOCATION_MARKERS_A +2026-09-22T20:41:06+03:00 pve01 homelab-backup-v2[1488093]: SELFTEST_REPLACEMENT486=PASS +OWNER_SNAPSHOT_END=A +OWNER_SNAPSHOT_BEGIN=B +OWNER_STATE_B|service=activating|sub=start|pid=1488093|invocation=567686e8c1b44972aa666eb7991b69d4|heavy=BUSY +1488093 1 17624 S 0.0 0.0 homelab-backup- /usr/bin/python3 /usr/local/sbin/homelab-backup-v2 scheduled +OWNER_WCHAN_B=poll_schedule_timeout.constprop.0 +OWNER_IO_B_rchar=3720682590 +OWNER_IO_B_wchar=2349144257 +OWNER_IO_B_syscr=1451910 +OWNER_IO_B_syscw=167139 +OWNER_IO_B_read_bytes=1454080 +OWNER_IO_B_write_bytes=988282880 +OWNER_PROCESS_TREE_B + 1273 1 2192590 S 0.0 0.0 sshd sshd: /usr/sbin/sshd -D [listener] 0 of 10-100 startups + 3566 3346 2192572 S 0.0 0.0 sshd sshd: /usr/sbin/sshd -D [listener] 0 of 10-100 startups + 4026 3805 2192569 S 0.0 0.0 sshd sshd: /usr/sbin/sshd -D [listener] 0 of 10-100 startups +1488093 1 17624 S 0.0 0.0 homelab-backup- /usr/bin/python3 /usr/local/sbin/homelab-backup-v2 scheduled +2468663 1273 30 S 0.0 0.0 sshd-session sshd-session: root [priv] +2468694 2468663 30 S 0.0 0.0 sshd-session sshd-session: root@notty +2469027 1488093 1 S 0.5 0.0 ssh /usr/bin/ssh -o BatchMode=yes -o StrictHostKeyChecking=yes root@pve02 PATH=/usr/sbin:/usr/bin:/sbin:/bin; export PATH; exec qm guest exec 160 --synchronous 1 --timeout 60 -- /usr/bin/python3 -I -B -c 'import base64,sys; p=sys.argv[1]; o=int(sys.argv[2]); n=int(sys.argv[3]); f=open(p,'"'"'rb'"'"'); f.seek(o); b=f.read(n); f.close(); sys.stdout.write(base64.b64encode(b).decode())' /tmp/homelab-v2-xf-0922T174106Ze0d1cc1923e5.tar.zst 988282880 131072 +2469118 2469117 0 S 0.0 0.0 sed sed -E -e s#(https?://)[^/@[:space:]]+@#\1[REDACTED]@#g -e s#(ssh://)[^/@[:space:]]+@#\1[REDACTED]@#g -e s#(git@[REDACTED]:[:space:]]+#\1[REDACTED]#g -e s/([0-9]{1,3}\.){3}[0-9]{1,3}/[PRIVATE_IP]/g -e s/[A-Za-z0-9._%+-]+@[A-Za-z0-9.-]+\.[A-Za-z]{2,}/[EMAIL]/g +HEAVY_LOCK_HOLDERS_B +CURRENT_INVOCATION_JOURNAL_B_LINES=1 +CURRENT_INVOCATION_JOURNAL_B_SHA256=99fbe57d7379e6a73b514b4c18ab500149de436d3635e4d1b604f7f9800d93bb +CURRENT_INVOCATION_MARKERS_B +2026-09-22T20:41:06+03:00 pve01 homelab-backup-v2[1488093]: SELFTEST_REPLACEMENT486=PASS +OWNER_SNAPSHOT_END=B +OWNER_DELTA_rchar=6308859 +OWNER_DELTA_wchar=4050236 +OWNER_DELTA_syscr=2502 +OWNER_DELTA_syscw=287 +OWNER_DELTA_read_bytes=0 +OWNER_DELTA_write_bytes=1703936 +OWNER_IO_PROGRESS_30S=true +OWNER_FINAL|service=activating|sub=start|pid=1488093|heavy=BUSY +AUDIT_RC=1 ISSUE|severity=HIGH|code=RPO_BREACH|logical_id=vm112|root_cause=NO_FRESH_V2_OR_SEALED_QUORUM ISSUE|severity=HIGH|code=RPO_BREACH|logical_id=vm113|root_cause=NO_FRESH_V2_OR_SEALED_QUORUM ISSUE|severity=HIGH|code=RPO_BREACH|logical_id=vm130|root_cause=NO_FRESH_V2_OR_SEALED_QUORUM @@ -72,30 +191,22 @@ ISSUE|severity=HIGH|code=RPO_BREACH|logical_id=vm160|root_cause=NO_FRESH_V2_OR_S ISSUE|severity=HIGH|code=RPO_BREACH|logical_id=vm170|root_cause=NO_FRESH_V2_OR_SEALED_QUORUM ISSUE|severity=HIGH|code=RPO_BREACH|logical_id=vm171|root_cause=NO_FRESH_V2_OR_SEALED_QUORUM ISSUE|severity=HIGH|code=RPO_BREACH|logical_id=vm190|root_cause=NO_FRESH_V2_OR_SEALED_QUORUM +ISSUE|severity=HIGH|code=RPO_BREACH|logical_id=vm9130|root_cause=NO_FRESH_V2_OR_SEALED_QUORUM ISSUE|severity=HIGH|code=LOCK_BUSY|logical_id=/run/homelab-backup-heavy.lock|root_cause=ACTIVE_OR_ORPHAN_LOCK -SUMMARY|AUDIT_STATUS=OK|BACKUP_STATUS=RED|ACTIVE_COUNT=16|ISSUE_COUNT=9|CLOUD_A=OK|CLOUD_B=OK|CLOUD_B_DRILL=OK|DR_PROVEN=YES|SOURCE_OFFHOST=OK -GLOBAL_HEAVY_LOCK_WAIT_BEGIN=true -GLOBAL_HEAVY_LOCK_WAIT_SECONDS=3600 -GLOBAL_HEAVY_LOCK_ACQUIRE_RC=1 -FAILURE=GLOBAL_HEAVY_LOCK_WAIT_TIMEOUT -TEMP_CLEANUP_ROLLBACK_RESTORED=true -TEMP_CLEANUP_LV_RESIDUE_COUNT=0 -CLEANUP_ACCEPTANCE=ROLLBACK_PROVEN -DECISION=HOLD_DR221_438_FINAL_RESTORE_INCOMPLETE -DR221_CLOSED=false -NEWFDOM_PROJECT_COMPLETE=false -FULL_PRODUCT_REMAINING=1 -NEXT_GATE=RECONCILE_ONLY_DR221_438_FAILURE -TASK_RESULT=HOLD_NEWFDOM_DR221_FINAL_RESTORE -CHANGES_MADE_BY_438=true -PRODUCT_MUTATION_BY_438=false -PRODUCTION_VM_CONFIG_MUTATION_BY_438=false -CLOUD_MUTATION_BY_438=false -TEMP_VM_MUTATION_BY_438=false -ROLLBACK_STARTED=true -ROLLBACK_RESTORED=true -HOMELAB_RESULT_CONTRACT={"version":1,"command_id":"NEWFI-260922-NEWFDOM-DR221-FINAL-ISOLATED-RESTORE438R1","status":"FAIL","changes_made":true,"rollback_started":true,"rollback_restored":true} -NEWFI438_END=true +SUMMARY|AUDIT_STATUS=OK|BACKUP_STATUS=RED|ACTIVE_COUNT=16|ISSUE_COUNT=10|CLOUD_A=OK|CLOUD_B=OK|CLOUD_B_DRILL=OK|DR_PROVEN=YES|SOURCE_OFFHOST=OK +AUDIT_NEWFDOM_ISSUE_COUNT=0 +AUDIT_UNEXPECTED_CODE_COUNT=0 +LOCK_RCA_CLASS=LEGITIMATE_LONG_OWNER_WITH_TIMER_ONLY_RETRIGGER_TOPOLOGY +SAFE_MAINTENANCE_COORDINATION=STOP_TIMER_ONLY_WAIT_CURRENT_SERVICE_NATURAL_EXIT_ACQUIRE_HEAVY_LOCK_RESTORE_TIMER_ON_ALL_PATHS +DECISION=PASS_DR221_439_SAFE_TIMER_MAINTENANCE_WINDOW_PROVEN +NEXT_GATE=FINAL_DR221_RESTORE_WITH_TEMPORARY_TIMER_PAUSE_NATURAL_OWNER_DRAIN_AND_GUARANTEED_TIMER_RESTORE +CHANGES_MADE_BY_439=false +PRODUCT_MUTATION_BY_439=false +VM_MUTATION_BY_439=false +CLOUD_MUTATION_BY_439=false +SYSTEMD_MUTATION_BY_439=false +HOMELAB_RESULT_CONTRACT={"version":1,"command_id":"NEWFI-260923-NEWFDOM-DR221-438-HEAVYLOCK-SCHEDULER-RCA439R1","status":"OK","changes_made":false,"rollback_started":false,"rollback_restored":null} +NEWFI439_END=true OUTPUT_END CHAT_OUTPUT_END