homelab-admin/homelab-public-context
1
0
Fork 0
Code Activity

runtime: publish NEWFI-260923-NEWFDOM-DR221-438-HEAVYLOCK-SCHEDULER-RCA439R1

Browse Source
This commit is contained in:
homelab-runtime-publisher
2026-09-22 22:34:54 +00:00
parent 00c99b6d05
commit bf9ef22d2e
4 changed files with 441 additions and 80 deletions
Download Patch File Download Diff File Expand all files Collapse all files
runtime/history/NEWFI-260923-NEWFDOM-DR221-438-HEAVYLOCK-SCHEDULER-RCA439R1.json
+38
View File
File diff suppressed because one or more lines are too long
runtime/history/NEWFI-260923-NEWFDOM-DR221-438-HEAVYLOCK-SCHEDULER-RCA439R1.txt
+212
View File
@@ -0,0 +1,212 @@
CHAT_OUTPUT_BEGIN
COMMAND_ID=NEWFI-260923-NEWFDOM-DR221-438-HEAVYLOCK-SCHEDULER-RCA439R1
STATUS=OK
RC=0
HOST=pve01
MODE=read-only
COMPONENT=newfi-dr221-438-heavylock-scheduler-rca
REFERENCE_REGISTER_CHECK=OK
REFERENCE_SHA256=5a3d8e5154c41cb582a4e0aca68090be1f0138918bf82131a948df326f9d8d66
ERROR_REGISTER_CHECK=OK
ERROR_REGISTER_SHA256=3b09a553ec0f527ed3afeed4753f52a74ee3036045bbfb3c685e9f8af4ba7ba0
COMMAND_SHA256=51f79da5a10e1f18db59f97482f73726947438de73dd916c8f50b0c75f336214
DUPLICATE_FAILED_COMMAND_BLOCKED=false
EXECUTION_STARTED=true
CHANGE_DECLARED=false
RESULT_CONTRACT_VALID=true
RESULT_CONTRACT_STATUS=NOT_APPLICABLE
RESULT_CONTRACT_ERROR=NONE
COMMAND_RC=0
CHANGES_MADE=false
ROLLBACK_STARTED=false
ROLLBACK_RESTORED=null
MUTATION_OUTCOME=NO_MUTATION
SANITIZED=yes
SECRETS_INCLUDED=no
PRIVATE_ADDRESSES_INCLUDED=no
RAW_EVIDENCE_SHA256=969f0ac21e8ea2afcc013eb098024f91c408e33d61cac081f888e40546c4b569
SANITIZED_OUTPUT_SHA256=969f0ac21e8ea2afcc013eb098024f91c408e33d61cac081f888e40546c4b569
OUTPUT_BEGIN
NEWFI439_BEGIN=true
COMMAND_ID=NEWFI-260923-NEWFDOM-DR221-438-HEAVYLOCK-SCHEDULER-RCA439R1
MODE=read-only
COMPONENT=newfi-dr221-438-heavylock-scheduler-rca
PREDECESSOR=NEWFI-260922-NEWFDOM-DR221-FINAL-ISOLATED-RESTORE438R1
TARGET_FAILURE=GLOBAL_HEAVY_LOCK_WAIT_TIMEOUT_3600S
MUTATION_BOUNDARY=NONE;SYSTEMD_TOPOLOGY_AND_OWNER_PROGRESS_READ_ONLY;NO_TIMER_ACTION;NO_SERVICE_ACTION;NO_SIGNAL;NO_LOCK_BREAK;NO_BACKUP_START;NO_VM_WRITE;NO_CLOUD_WRITE
REFERENCE_SHA256=5a3d8e5154c41cb582a4e0aca68090be1f0138918bf82131a948df326f9d8d66
ERROR_REGISTER_SHA256=3b09a553ec0f527ed3afeed4753f52a74ee3036045bbfb3c685e9f8af4ba7ba0
RUNNER_SHA256=b248a4c32c9cc64e5747e7dce6c7fc0a23f5124a77c71ce72e27a81aceae9d2d
FRESHNESS_SHA256=2a9b391e05fd38755f18d2c920ded3758cf162fdb332ab84203475f9017d3558
AUDIT_SHA256=f7b936fa997bdefdd0c3c5d1d96e496888ebebc8ea85a60ed3bf5bcc9bf4962a
WRAPPER_SHA256=5077444065c732451cb84898680f62361b21a24ef9eb4f191253e82ead524ea2
VM210_FRESHNESS_RC=0
FRESHNESS_STATUS=OK
VMID=210
DECISION=SEALED_PROVEN
GENERATION=vm210-pve01-20260922T172531Z
UPDATED_AT_UTC=2026-09-22T17:27:10Z
SEALED_EPOCH=1790098030
STATE_FILE=/var/lib/homelab-backup/cloud-quorum/vm210/vm210-pve01-20260922T172531Z.json
RESTORE_EVIDENCE=/var/lib/homelab-backup/vm-restore-validation/vm210/vm210-pve01-20260922T172531Z.json
VM210_SEALED_PROVEN_CURRENT=true
SYSTEMD_TOPOLOGY_BEGIN=true
Type=oneshot
Restart=no
MainPID=1488093
Result=success
ExecMainStartTimestamp=Tue 2026-09-22 20:41:06 MSK
ExecMainExitTimestamp=
ExecMainStatus=0
Triggers=
TriggeredBy=homelab-backup-v2.timer
LoadState=loaded
ActiveState=activating
SubState=start
UnitFileState=static
CanStop=yes
RefuseManualStop=no
InvocationID=567686e8c1b44972aa666eb7991b69d4
Unit=homelab-backup-v2.service
NextElapseUSecRealtime=
LastTriggerUSec=Tue 2026-09-22 20:41:06 MSK
AccuracyUSec=1min
RandomizedDelayUSec=10min
Persistent=no
Triggers=homelab-backup-v2.service
LoadState=loaded
ActiveState=active
SubState=running
UnitFileState=enabled
CanStop=yes
RefuseManualStop=no
SERVICE_UNIT_SAFE_LINES_BEGIN=true
# /etc/systemd/system/homelab-backup-v2.service
[Unit]
Description=Homelab Backup V2 manifest-driven scheduler
After=network-online.target
Wants=network-online.target
[Service]
Type=oneshot
User=root
Group=root
UMask=0077
ExecStart=/usr/local/sbin/homelab-backup-v2 scheduled
Nice=10
IOSchedulingClass=best-effort
IOSchedulingPriority=7
TimeoutStartSec=8h
SERVICE_UNIT_SAFE_LINES_END=true
TIMER_UNIT_SAFE_LINES_BEGIN=true
# /etc/systemd/system/homelab-backup-v2.timer
[Unit]
Description=Homelab Backup V2 scheduler timer
[Timer]
OnActiveSec=20min
OnUnitActiveSec=30min
RandomizedDelaySec=10min
Unit=homelab-backup-v2.service
[Install]
WantedBy=timers.target
TIMER_UNIT_SAFE_LINES_END=true
REVERSE_TRIGGER_UNITS_BEGIN=true
homelab-backup-v2.service
REVERSE_TRIGGER_UNITS_END=true
TIMER_ENABLED=enabled
TIMER_ACTIVE=active
SERVICE_UNITFILE_STATE=static
SERVICE_RESTART=no
SERVICE_TYPE=oneshot
SERVICE_TRIGGERED_BY=homelab-backup-v2.timer
TIMER_TRIGGERS=homelab-backup-v2.service
TIMER_ONLY_MAINTENANCE_WINDOW_TOPOLOGY=true
SYSTEMD_TOPOLOGY_END=true
OWNER_SNAPSHOT_BEGIN=A
OWNER_STATE_A|service=activating|sub=start|pid=1488093|invocation=567686e8c1b44972aa666eb7991b69d4|heavy=BUSY
1488093 1 17594 S 0.0 0.0 homelab-backup- /usr/bin/python3 /usr/local/sbin/homelab-backup-v2 scheduled
OWNER_WCHAN_A=poll_schedule_timeout.constprop.0
OWNER_IO_A_rchar=3714373731
OWNER_IO_A_wchar=2345094021
OWNER_IO_A_syscr=1449408
OWNER_IO_A_syscw=166852
OWNER_IO_A_read_bytes=1454080
OWNER_IO_A_write_bytes=986578944
OWNER_PROCESS_TREE_A
1273 1 2192560 S 0.0 0.0 sshd sshd: /usr/sbin/sshd -D [listener] 0 of 10-100 startups
3566 3346 2192542 S 0.0 0.0 sshd sshd: /usr/sbin/sshd -D [listener] 0 of 10-100 startups
4026 3805 2192539 S 0.0 0.0 sshd sshd: /usr/sbin/sshd -D [listener] 0 of 10-100 startups
1488093 1 17594 S 0.0 0.0 homelab-backup- /usr/bin/python3 /usr/local/sbin/homelab-backup-v2 scheduled
2468623 1488093 1 S 0.5 0.0 ssh /usr/bin/ssh -o BatchMode=yes -o StrictHostKeyChecking=yes root@pve02 PATH=/usr/sbin:/usr/bin:/sbin:/bin; export PATH; exec qm guest exec 160 --synchronous 1 --timeout 60 -- /usr/bin/python3 -I -B -c 'import base64,sys; p=sys.argv[1]; o=int(sys.argv[2]); n=int(sys.argv[3]); f=open(p,'"'"'rb'"'"'); f.seek(o); b=f.read(n); f.close(); sys.stdout.write(base64.b64encode(b).decode())' /tmp/homelab-v2-xf-0922T174106Ze0d1cc1923e5.tar.zst 986578944 131072
2468663 1273 0 S 0.0 0.0 sshd-session sshd-session: root [priv]
2468694 2468663 0 S 0.0 0.0 sshd-session sshd-session: root@notty
2468806 2468805 0 S 0.0 0.0 sed sed -E -e s#(https?://)[^/@[:space:]]+@#\1[REDACTED]@#g -e s#(ssh://)[^/@[:space:]]+@#\1[REDACTED]@#g -e s#(git@[REDACTED]:[:space:]]+#\1[REDACTED]#g -e s/([0-9]{1,3}\.){3}[0-9]{1,3}/[PRIVATE_IP]/g -e s/[A-Za-z0-9._%+-]+@[A-Za-z0-9.-]+\.[A-Za-z]{2,}/[EMAIL]/g
HEAVY_LOCK_HOLDERS_A
CURRENT_INVOCATION_JOURNAL_A_LINES=1
CURRENT_INVOCATION_JOURNAL_A_SHA256=99fbe57d7379e6a73b514b4c18ab500149de436d3635e4d1b604f7f9800d93bb
CURRENT_INVOCATION_MARKERS_A
2026-09-22T20:41:06+03:00 pve01 homelab-backup-v2[1488093]: SELFTEST_REPLACEMENT486=PASS
OWNER_SNAPSHOT_END=A
OWNER_SNAPSHOT_BEGIN=B
OWNER_STATE_B|service=activating|sub=start|pid=1488093|invocation=567686e8c1b44972aa666eb7991b69d4|heavy=BUSY
1488093 1 17624 S 0.0 0.0 homelab-backup- /usr/bin/python3 /usr/local/sbin/homelab-backup-v2 scheduled
OWNER_WCHAN_B=poll_schedule_timeout.constprop.0
OWNER_IO_B_rchar=3720682590
OWNER_IO_B_wchar=2349144257
OWNER_IO_B_syscr=1451910
OWNER_IO_B_syscw=167139
OWNER_IO_B_read_bytes=1454080
OWNER_IO_B_write_bytes=988282880
OWNER_PROCESS_TREE_B
1273 1 2192590 S 0.0 0.0 sshd sshd: /usr/sbin/sshd -D [listener] 0 of 10-100 startups
3566 3346 2192572 S 0.0 0.0 sshd sshd: /usr/sbin/sshd -D [listener] 0 of 10-100 startups
4026 3805 2192569 S 0.0 0.0 sshd sshd: /usr/sbin/sshd -D [listener] 0 of 10-100 startups
1488093 1 17624 S 0.0 0.0 homelab-backup- /usr/bin/python3 /usr/local/sbin/homelab-backup-v2 scheduled
2468663 1273 30 S 0.0 0.0 sshd-session sshd-session: root [priv]
2468694 2468663 30 S 0.0 0.0 sshd-session sshd-session: root@notty
2469027 1488093 1 S 0.5 0.0 ssh /usr/bin/ssh -o BatchMode=yes -o StrictHostKeyChecking=yes root@pve02 PATH=/usr/sbin:/usr/bin:/sbin:/bin; export PATH; exec qm guest exec 160 --synchronous 1 --timeout 60 -- /usr/bin/python3 -I -B -c 'import base64,sys; p=sys.argv[1]; o=int(sys.argv[2]); n=int(sys.argv[3]); f=open(p,'"'"'rb'"'"'); f.seek(o); b=f.read(n); f.close(); sys.stdout.write(base64.b64encode(b).decode())' /tmp/homelab-v2-xf-0922T174106Ze0d1cc1923e5.tar.zst 988282880 131072
2469118 2469117 0 S 0.0 0.0 sed sed -E -e s#(https?://)[^/@[:space:]]+@#\1[REDACTED]@#g -e s#(ssh://)[^/@[:space:]]+@#\1[REDACTED]@#g -e s#(git@[REDACTED]:[:space:]]+#\1[REDACTED]#g -e s/([0-9]{1,3}\.){3}[0-9]{1,3}/[PRIVATE_IP]/g -e s/[A-Za-z0-9._%+-]+@[A-Za-z0-9.-]+\.[A-Za-z]{2,}/[EMAIL]/g
HEAVY_LOCK_HOLDERS_B
CURRENT_INVOCATION_JOURNAL_B_LINES=1
CURRENT_INVOCATION_JOURNAL_B_SHA256=99fbe57d7379e6a73b514b4c18ab500149de436d3635e4d1b604f7f9800d93bb
CURRENT_INVOCATION_MARKERS_B
2026-09-22T20:41:06+03:00 pve01 homelab-backup-v2[1488093]: SELFTEST_REPLACEMENT486=PASS
OWNER_SNAPSHOT_END=B
OWNER_DELTA_rchar=6308859
OWNER_DELTA_wchar=4050236
OWNER_DELTA_syscr=2502
OWNER_DELTA_syscw=287
OWNER_DELTA_read_bytes=0
OWNER_DELTA_write_bytes=1703936
OWNER_IO_PROGRESS_30S=true
OWNER_FINAL|service=activating|sub=start|pid=1488093|heavy=BUSY
AUDIT_RC=1
ISSUE|severity=HIGH|code=RPO_BREACH|logical_id=vm112|root_cause=NO_FRESH_V2_OR_SEALED_QUORUM
ISSUE|severity=HIGH|code=RPO_BREACH|logical_id=vm113|root_cause=NO_FRESH_V2_OR_SEALED_QUORUM
ISSUE|severity=HIGH|code=RPO_BREACH|logical_id=vm130|root_cause=NO_FRESH_V2_OR_SEALED_QUORUM
ISSUE|severity=HIGH|code=RPO_BREACH|logical_id=vm150|root_cause=NO_FRESH_V2_OR_SEALED_QUORUM
ISSUE|severity=HIGH|code=RPO_BREACH|logical_id=vm160|root_cause=NO_FRESH_V2_OR_SEALED_QUORUM
ISSUE|severity=HIGH|code=RPO_BREACH|logical_id=vm170|root_cause=NO_FRESH_V2_OR_SEALED_QUORUM
ISSUE|severity=HIGH|code=RPO_BREACH|logical_id=vm171|root_cause=NO_FRESH_V2_OR_SEALED_QUORUM
ISSUE|severity=HIGH|code=RPO_BREACH|logical_id=vm190|root_cause=NO_FRESH_V2_OR_SEALED_QUORUM
ISSUE|severity=HIGH|code=RPO_BREACH|logical_id=vm9130|root_cause=NO_FRESH_V2_OR_SEALED_QUORUM
ISSUE|severity=HIGH|code=LOCK_BUSY|logical_id=/run/homelab-backup-heavy.lock|root_cause=ACTIVE_OR_ORPHAN_LOCK
SUMMARY|AUDIT_STATUS=OK|BACKUP_STATUS=RED|ACTIVE_COUNT=16|ISSUE_COUNT=10|CLOUD_A=OK|CLOUD_B=OK|CLOUD_B_DRILL=OK|DR_PROVEN=YES|SOURCE_OFFHOST=OK
AUDIT_NEWFDOM_ISSUE_COUNT=0
AUDIT_UNEXPECTED_CODE_COUNT=0
LOCK_RCA_CLASS=LEGITIMATE_LONG_OWNER_WITH_TIMER_ONLY_RETRIGGER_TOPOLOGY
SAFE_MAINTENANCE_COORDINATION=STOP_TIMER_ONLY_WAIT_CURRENT_SERVICE_NATURAL_EXIT_ACQUIRE_HEAVY_LOCK_RESTORE_TIMER_ON_ALL_PATHS
DECISION=PASS_DR221_439_SAFE_TIMER_MAINTENANCE_WINDOW_PROVEN
NEXT_GATE=FINAL_DR221_RESTORE_WITH_TEMPORARY_TIMER_PAUSE_NATURAL_OWNER_DRAIN_AND_GUARANTEED_TIMER_RESTORE
CHANGES_MADE_BY_439=false
PRODUCT_MUTATION_BY_439=false
VM_MUTATION_BY_439=false
CLOUD_MUTATION_BY_439=false
SYSTEMD_MUTATION_BY_439=false
HOMELAB_RESULT_CONTRACT={"version":1,"command_id":"NEWFI-260923-NEWFDOM-DR221-438-HEAVYLOCK-SCHEDULER-RCA439R1","status":"OK","changes_made":false,"rollback_started":false,"rollback_restored":null}
NEWFI439_END=true
OUTPUT_END
CHAT_OUTPUT_END
runtime/latest.json
+18 -18
View File
File diff suppressed because one or more lines are too long
runtime/latest.txt
+173 -62
View File
@@ -1,56 +1,46 @@
CHAT_OUTPUT_BEGIN
COMMAND_ID=NEWFI-260922-NEWFDOM-DR221-FINAL-ISOLATED-RESTORE438R1
STATUS=FAIL
RC=5
COMMAND_ID=NEWFI-260923-NEWFDOM-DR221-438-HEAVYLOCK-SCHEDULER-RCA439R1
STATUS=OK
RC=0
HOST=pve01
MODE=change
COMPONENT=newfi-dr221-final-isolated-restore-acceptance-437
MODE=read-only
COMPONENT=newfi-dr221-438-heavylock-scheduler-rca
REFERENCE_REGISTER_CHECK=OK
REFERENCE_SHA256=5a3d8e5154c41cb582a4e0aca68090be1f0138918bf82131a948df326f9d8d66
ERROR_REGISTER_CHECK=OK
ERROR_REGISTER_SHA256=3b09a553ec0f527ed3afeed4753f52a74ee3036045bbfb3c685e9f8af4ba7ba0
COMMAND_SHA256=c40ca75d93f5487b08adf55b1efc96d566b0164435923b2bb84485ed2862a342
COMMAND_SHA256=51f79da5a10e1f18db59f97482f73726947438de73dd916c8f50b0c75f336214
DUPLICATE_FAILED_COMMAND_BLOCKED=false
EXECUTION_STARTED=true
CHANGE_DECLARED=true
CHANGE_DECLARED=false
RESULT_CONTRACT_VALID=true
RESULT_CONTRACT_STATUS=FAIL
RESULT_CONTRACT_STATUS=NOT_APPLICABLE
RESULT_CONTRACT_ERROR=NONE
COMMAND_RC=5
CHANGES_MADE=true
ROLLBACK_STARTED=true
ROLLBACK_RESTORED=true
MUTATION_OUTCOME=ROLLED_BACK
COMMAND_RC=0
CHANGES_MADE=false
ROLLBACK_STARTED=false
ROLLBACK_RESTORED=null
MUTATION_OUTCOME=NO_MUTATION
SANITIZED=yes
SECRETS_INCLUDED=no
PRIVATE_ADDRESSES_INCLUDED=no
RAW_EVIDENCE_SHA256=79e238f968290832f27370590912a9923c609c80dd3c71f046ccaa7e70c053de
SANITIZED_OUTPUT_SHA256=79e238f968290832f27370590912a9923c609c80dd3c71f046ccaa7e70c053de
RAW_EVIDENCE_SHA256=969f0ac21e8ea2afcc013eb098024f91c408e33d61cac081f888e40546c4b569
SANITIZED_OUTPUT_SHA256=969f0ac21e8ea2afcc013eb098024f91c408e33d61cac081f888e40546c4b569
OUTPUT_BEGIN
NEWFI438_BEGIN=true
COMMAND_ID=NEWFI-260922-NEWFDOM-DR221-FINAL-ISOLATED-RESTORE438R1
MODE=change
COMPONENT=newfi-dr221-final-isolated-restore-acceptance-437
ANCHOR_425_FIRST_COMMIT=595fac75a29e35f975cda1a51fab6919fce528d8
PREDECESSOR_437=PASS_DR221_437_FINAL_SOURCE_PRODUCT_CONTRACT_PROVEN
DR221_BACKUP_GENERATION=vm210-pve01-20260922T172531Z
TEST_VMID=9210
AUDIT_POLICY=CORE_HEALTH_PLUS_NO_NEW_UNEXPECTED_OR_NEWFDOM_ISSUES;RPO_BACKLOG_EXTERNAL
MUTATION_BOUNDARY=CLOUD_A_READ_ONLY_RECONSTRUCTION;TEMP_VM_RESTORE_9210_ONLY;TEMP_VM_NIC_ISOLATION;TEMP_BOOT_QGA_VERIFY;TEMP_DESTROY;LOCAL_FULL_RESTORE_EVIDENCE;NO_PRODUCTION_VM_CONFIG_WRITE;NO_CLOUD_WRITE_OR_DELETE
NEWFI439_BEGIN=true
COMMAND_ID=NEWFI-260923-NEWFDOM-DR221-438-HEAVYLOCK-SCHEDULER-RCA439R1
MODE=read-only
COMPONENT=newfi-dr221-438-heavylock-scheduler-rca
PREDECESSOR=NEWFI-260922-NEWFDOM-DR221-FINAL-ISOLATED-RESTORE438R1
TARGET_FAILURE=GLOBAL_HEAVY_LOCK_WAIT_TIMEOUT_3600S
MUTATION_BOUNDARY=NONE;SYSTEMD_TOPOLOGY_AND_OWNER_PROGRESS_READ_ONLY;NO_TIMER_ACTION;NO_SERVICE_ACTION;NO_SIGNAL;NO_LOCK_BREAK;NO_BACKUP_START;NO_VM_WRITE;NO_CLOUD_WRITE
REFERENCE_SHA256=5a3d8e5154c41cb582a4e0aca68090be1f0138918bf82131a948df326f9d8d66
ERROR_REGISTER_SHA256=3b09a553ec0f527ed3afeed4753f52a74ee3036045bbfb3c685e9f8af4ba7ba0
RUNNER_SHA256=b248a4c32c9cc64e5747e7dce6c7fc0a23f5124a77c71ce72e27a81aceae9d2d
QUEUE_SHA256=96280069bbe3c4930105f8f00e49a299a24da299eb84f2c0a43360afa4c9713e
FRESHNESS_SHA256=2a9b391e05fd38755f18d2c920ded3758cf162fdb332ab84203475f9017d3558
WORKER_SHA256=4c729debe80f99528fea1f6e1723e7c4d03c5249ba01fd640aaa7c4a4075075c
AUDIT_SHA256=f7b936fa997bdefdd0c3c5d1d96e496888ebebc8ea85a60ed3bf5bcc9bf4962a
WORKER_PLAN_RC=0
PLAN_STATUS=OK
VMID=210
OWNER=pve01
TYPE=qemu
EXPECTED_STATUS=running
FRESHNESS_RC=0
WRAPPER_SHA256=5077444065c732451cb84898680f62361b21a24ef9eb4f191253e82ead524ea2
VM210_FRESHNESS_RC=0
FRESHNESS_STATUS=OK
VMID=210
DECISION=SEALED_PROVEN
@@ -59,11 +49,140 @@ UPDATED_AT_UTC=2026-09-22T17:27:10Z
SEALED_EPOCH=1790098030
STATE_FILE=/var/lib/homelab-backup/cloud-quorum/vm210/vm210-pve01-20260922T172531Z.json
RESTORE_EVIDENCE=/var/lib/homelab-backup/vm-restore-validation/vm210/vm210-pve01-20260922T172531Z.json
SOURCE_AGENT_PING_RC=0
SOURCE_PRODUCT_TRANSPORT_RC=0
SOURCE_PRODUCT_GUEST|exited=1|exitcode=0
SOURCE_PRODUCT_SIGNATURE_SHA256=fcb934facb4ce56d534ba49759b29a4f7c59a39848a779186ba325ecbad9c02a
AUDIT_ENTRY_RC=1
VM210_SEALED_PROVEN_CURRENT=true
SYSTEMD_TOPOLOGY_BEGIN=true
Type=oneshot
Restart=no
MainPID=1488093
Result=success
ExecMainStartTimestamp=Tue 2026-09-22 20:41:06 MSK
ExecMainExitTimestamp=
ExecMainStatus=0
Triggers=
TriggeredBy=homelab-backup-v2.timer
LoadState=loaded
ActiveState=activating
SubState=start
UnitFileState=static
CanStop=yes
RefuseManualStop=no
InvocationID=567686e8c1b44972aa666eb7991b69d4
Unit=homelab-backup-v2.service
NextElapseUSecRealtime=
LastTriggerUSec=Tue 2026-09-22 20:41:06 MSK
AccuracyUSec=1min
RandomizedDelayUSec=10min
Persistent=no
Triggers=homelab-backup-v2.service
LoadState=loaded
ActiveState=active
SubState=running
UnitFileState=enabled
CanStop=yes
RefuseManualStop=no
SERVICE_UNIT_SAFE_LINES_BEGIN=true
# /etc/systemd/system/homelab-backup-v2.service
[Unit]
Description=Homelab Backup V2 manifest-driven scheduler
After=network-online.target
Wants=network-online.target
[Service]
Type=oneshot
User=root
Group=root
UMask=0077
ExecStart=/usr/local/sbin/homelab-backup-v2 scheduled
Nice=10
IOSchedulingClass=best-effort
IOSchedulingPriority=7
TimeoutStartSec=8h
SERVICE_UNIT_SAFE_LINES_END=true
TIMER_UNIT_SAFE_LINES_BEGIN=true
# /etc/systemd/system/homelab-backup-v2.timer
[Unit]
Description=Homelab Backup V2 scheduler timer
[Timer]
OnActiveSec=20min
OnUnitActiveSec=30min
RandomizedDelaySec=10min
Unit=homelab-backup-v2.service
[Install]
WantedBy=timers.target
TIMER_UNIT_SAFE_LINES_END=true
REVERSE_TRIGGER_UNITS_BEGIN=true
homelab-backup-v2.service
REVERSE_TRIGGER_UNITS_END=true
TIMER_ENABLED=enabled
TIMER_ACTIVE=active
SERVICE_UNITFILE_STATE=static
SERVICE_RESTART=no
SERVICE_TYPE=oneshot
SERVICE_TRIGGERED_BY=homelab-backup-v2.timer
TIMER_TRIGGERS=homelab-backup-v2.service
TIMER_ONLY_MAINTENANCE_WINDOW_TOPOLOGY=true
SYSTEMD_TOPOLOGY_END=true
OWNER_SNAPSHOT_BEGIN=A
OWNER_STATE_A|service=activating|sub=start|pid=1488093|invocation=567686e8c1b44972aa666eb7991b69d4|heavy=BUSY
1488093 1 17594 S 0.0 0.0 homelab-backup- /usr/bin/python3 /usr/local/sbin/homelab-backup-v2 scheduled
OWNER_WCHAN_A=poll_schedule_timeout.constprop.0
OWNER_IO_A_rchar=3714373731
OWNER_IO_A_wchar=2345094021
OWNER_IO_A_syscr=1449408
OWNER_IO_A_syscw=166852
OWNER_IO_A_read_bytes=1454080
OWNER_IO_A_write_bytes=986578944
OWNER_PROCESS_TREE_A
1273 1 2192560 S 0.0 0.0 sshd sshd: /usr/sbin/sshd -D [listener] 0 of 10-100 startups
3566 3346 2192542 S 0.0 0.0 sshd sshd: /usr/sbin/sshd -D [listener] 0 of 10-100 startups
4026 3805 2192539 S 0.0 0.0 sshd sshd: /usr/sbin/sshd -D [listener] 0 of 10-100 startups
1488093 1 17594 S 0.0 0.0 homelab-backup- /usr/bin/python3 /usr/local/sbin/homelab-backup-v2 scheduled
2468623 1488093 1 S 0.5 0.0 ssh /usr/bin/ssh -o BatchMode=yes -o StrictHostKeyChecking=yes root@pve02 PATH=/usr/sbin:/usr/bin:/sbin:/bin; export PATH; exec qm guest exec 160 --synchronous 1 --timeout 60 -- /usr/bin/python3 -I -B -c 'import base64,sys; p=sys.argv[1]; o=int(sys.argv[2]); n=int(sys.argv[3]); f=open(p,'"'"'rb'"'"'); f.seek(o); b=f.read(n); f.close(); sys.stdout.write(base64.b64encode(b).decode())' /tmp/homelab-v2-xf-0922T174106Ze0d1cc1923e5.tar.zst 986578944 131072
2468663 1273 0 S 0.0 0.0 sshd-session sshd-session: root [priv]
2468694 2468663 0 S 0.0 0.0 sshd-session sshd-session: root@notty
2468806 2468805 0 S 0.0 0.0 sed sed -E -e s#(https?://)[^/@[:space:]]+@#\1[REDACTED]@#g -e s#(ssh://)[^/@[:space:]]+@#\1[REDACTED]@#g -e s#(git@[REDACTED]:[:space:]]+#\1[REDACTED]#g -e s/([0-9]{1,3}\.){3}[0-9]{1,3}/[PRIVATE_IP]/g -e s/[A-Za-z0-9._%+-]+@[A-Za-z0-9.-]+\.[A-Za-z]{2,}/[EMAIL]/g
HEAVY_LOCK_HOLDERS_A
CURRENT_INVOCATION_JOURNAL_A_LINES=1
CURRENT_INVOCATION_JOURNAL_A_SHA256=99fbe57d7379e6a73b514b4c18ab500149de436d3635e4d1b604f7f9800d93bb
CURRENT_INVOCATION_MARKERS_A
2026-09-22T20:41:06+03:00 pve01 homelab-backup-v2[1488093]: SELFTEST_REPLACEMENT486=PASS
OWNER_SNAPSHOT_END=A
OWNER_SNAPSHOT_BEGIN=B
OWNER_STATE_B|service=activating|sub=start|pid=1488093|invocation=567686e8c1b44972aa666eb7991b69d4|heavy=BUSY
1488093 1 17624 S 0.0 0.0 homelab-backup- /usr/bin/python3 /usr/local/sbin/homelab-backup-v2 scheduled
OWNER_WCHAN_B=poll_schedule_timeout.constprop.0
OWNER_IO_B_rchar=3720682590
OWNER_IO_B_wchar=2349144257
OWNER_IO_B_syscr=1451910
OWNER_IO_B_syscw=167139
OWNER_IO_B_read_bytes=1454080
OWNER_IO_B_write_bytes=988282880
OWNER_PROCESS_TREE_B
1273 1 2192590 S 0.0 0.0 sshd sshd: /usr/sbin/sshd -D [listener] 0 of 10-100 startups
3566 3346 2192572 S 0.0 0.0 sshd sshd: /usr/sbin/sshd -D [listener] 0 of 10-100 startups
4026 3805 2192569 S 0.0 0.0 sshd sshd: /usr/sbin/sshd -D [listener] 0 of 10-100 startups
1488093 1 17624 S 0.0 0.0 homelab-backup- /usr/bin/python3 /usr/local/sbin/homelab-backup-v2 scheduled
2468663 1273 30 S 0.0 0.0 sshd-session sshd-session: root [priv]
2468694 2468663 30 S 0.0 0.0 sshd-session sshd-session: root@notty
2469027 1488093 1 S 0.5 0.0 ssh /usr/bin/ssh -o BatchMode=yes -o StrictHostKeyChecking=yes root@pve02 PATH=/usr/sbin:/usr/bin:/sbin:/bin; export PATH; exec qm guest exec 160 --synchronous 1 --timeout 60 -- /usr/bin/python3 -I -B -c 'import base64,sys; p=sys.argv[1]; o=int(sys.argv[2]); n=int(sys.argv[3]); f=open(p,'"'"'rb'"'"'); f.seek(o); b=f.read(n); f.close(); sys.stdout.write(base64.b64encode(b).decode())' /tmp/homelab-v2-xf-0922T174106Ze0d1cc1923e5.tar.zst 988282880 131072
2469118 2469117 0 S 0.0 0.0 sed sed -E -e s#(https?://)[^/@[:space:]]+@#\1[REDACTED]@#g -e s#(ssh://)[^/@[:space:]]+@#\1[REDACTED]@#g -e s#(git@[REDACTED]:[:space:]]+#\1[REDACTED]#g -e s/([0-9]{1,3}\.){3}[0-9]{1,3}/[PRIVATE_IP]/g -e s/[A-Za-z0-9._%+-]+@[A-Za-z0-9.-]+\.[A-Za-z]{2,}/[EMAIL]/g
HEAVY_LOCK_HOLDERS_B
CURRENT_INVOCATION_JOURNAL_B_LINES=1
CURRENT_INVOCATION_JOURNAL_B_SHA256=99fbe57d7379e6a73b514b4c18ab500149de436d3635e4d1b604f7f9800d93bb
CURRENT_INVOCATION_MARKERS_B
2026-09-22T20:41:06+03:00 pve01 homelab-backup-v2[1488093]: SELFTEST_REPLACEMENT486=PASS
OWNER_SNAPSHOT_END=B
OWNER_DELTA_rchar=6308859
OWNER_DELTA_wchar=4050236
OWNER_DELTA_syscr=2502
OWNER_DELTA_syscw=287
OWNER_DELTA_read_bytes=0
OWNER_DELTA_write_bytes=1703936
OWNER_IO_PROGRESS_30S=true
OWNER_FINAL|service=activating|sub=start|pid=1488093|heavy=BUSY
AUDIT_RC=1
ISSUE|severity=HIGH|code=RPO_BREACH|logical_id=vm112|root_cause=NO_FRESH_V2_OR_SEALED_QUORUM
ISSUE|severity=HIGH|code=RPO_BREACH|logical_id=vm113|root_cause=NO_FRESH_V2_OR_SEALED_QUORUM
ISSUE|severity=HIGH|code=RPO_BREACH|logical_id=vm130|root_cause=NO_FRESH_V2_OR_SEALED_QUORUM
@@ -72,30 +191,22 @@ ISSUE|severity=HIGH|code=RPO_BREACH|logical_id=vm160|root_cause=NO_FRESH_V2_OR_S
ISSUE|severity=HIGH|code=RPO_BREACH|logical_id=vm170|root_cause=NO_FRESH_V2_OR_SEALED_QUORUM
ISSUE|severity=HIGH|code=RPO_BREACH|logical_id=vm171|root_cause=NO_FRESH_V2_OR_SEALED_QUORUM
ISSUE|severity=HIGH|code=RPO_BREACH|logical_id=vm190|root_cause=NO_FRESH_V2_OR_SEALED_QUORUM
ISSUE|severity=HIGH|code=RPO_BREACH|logical_id=vm9130|root_cause=NO_FRESH_V2_OR_SEALED_QUORUM
ISSUE|severity=HIGH|code=LOCK_BUSY|logical_id=/run/homelab-backup-heavy.lock|root_cause=ACTIVE_OR_ORPHAN_LOCK
SUMMARY|AUDIT_STATUS=OK|BACKUP_STATUS=RED|ACTIVE_COUNT=16|ISSUE_COUNT=9|CLOUD_A=OK|CLOUD_B=OK|CLOUD_B_DRILL=OK|DR_PROVEN=YES|SOURCE_OFFHOST=OK
GLOBAL_HEAVY_LOCK_WAIT_BEGIN=true
GLOBAL_HEAVY_LOCK_WAIT_SECONDS=3600
GLOBAL_HEAVY_LOCK_ACQUIRE_RC=1
FAILURE=GLOBAL_HEAVY_LOCK_WAIT_TIMEOUT
TEMP_CLEANUP_ROLLBACK_RESTORED=true
TEMP_CLEANUP_LV_RESIDUE_COUNT=0
CLEANUP_ACCEPTANCE=ROLLBACK_PROVEN
DECISION=HOLD_DR221_438_FINAL_RESTORE_INCOMPLETE
DR221_CLOSED=false
NEWFDOM_PROJECT_COMPLETE=false
FULL_PRODUCT_REMAINING=1
NEXT_GATE=RECONCILE_ONLY_DR221_438_FAILURE
TASK_RESULT=HOLD_NEWFDOM_DR221_FINAL_RESTORE
CHANGES_MADE_BY_438=true
PRODUCT_MUTATION_BY_438=false
PRODUCTION_VM_CONFIG_MUTATION_BY_438=false
CLOUD_MUTATION_BY_438=false
TEMP_VM_MUTATION_BY_438=false
ROLLBACK_STARTED=true
ROLLBACK_RESTORED=true
HOMELAB_RESULT_CONTRACT={"version":1,"command_id":"NEWFI-260922-NEWFDOM-DR221-FINAL-ISOLATED-RESTORE438R1","status":"FAIL","changes_made":true,"rollback_started":true,"rollback_restored":true}
NEWFI438_END=true
SUMMARY|AUDIT_STATUS=OK|BACKUP_STATUS=RED|ACTIVE_COUNT=16|ISSUE_COUNT=10|CLOUD_A=OK|CLOUD_B=OK|CLOUD_B_DRILL=OK|DR_PROVEN=YES|SOURCE_OFFHOST=OK
AUDIT_NEWFDOM_ISSUE_COUNT=0
AUDIT_UNEXPECTED_CODE_COUNT=0
LOCK_RCA_CLASS=LEGITIMATE_LONG_OWNER_WITH_TIMER_ONLY_RETRIGGER_TOPOLOGY
SAFE_MAINTENANCE_COORDINATION=STOP_TIMER_ONLY_WAIT_CURRENT_SERVICE_NATURAL_EXIT_ACQUIRE_HEAVY_LOCK_RESTORE_TIMER_ON_ALL_PATHS
DECISION=PASS_DR221_439_SAFE_TIMER_MAINTENANCE_WINDOW_PROVEN
NEXT_GATE=FINAL_DR221_RESTORE_WITH_TEMPORARY_TIMER_PAUSE_NATURAL_OWNER_DRAIN_AND_GUARANTEED_TIMER_RESTORE
CHANGES_MADE_BY_439=false
PRODUCT_MUTATION_BY_439=false
VM_MUTATION_BY_439=false
CLOUD_MUTATION_BY_439=false
SYSTEMD_MUTATION_BY_439=false
HOMELAB_RESULT_CONTRACT={"version":1,"command_id":"NEWFI-260923-NEWFDOM-DR221-438-HEAVYLOCK-SCHEDULER-RCA439R1","status":"OK","changes_made":false,"rollback_started":false,"rollback_restored":null}
NEWFI439_END=true
OUTPUT_END
CHAT_OUTPUT_END