diff --git a/runtime/history/SUPPORT-260924-HOMELAB-MOLDOVA-BACKUP-SSH-EXECUTOR-RCA-1145R1.json b/runtime/history/SUPPORT-260924-HOMELAB-MOLDOVA-BACKUP-SSH-EXECUTOR-RCA-1145R1.json new file mode 100644 index 00000000..e5b31ac8 --- /dev/null +++ b/runtime/history/SUPPORT-260924-HOMELAB-MOLDOVA-BACKUP-SSH-EXECUTOR-RCA-1145R1.json @@ -0,0 +1,38 @@ +{ + "schema_version": 1, + "channel": "homelab-runtime", + "command_id": "SUPPORT-260924-HOMELAB-MOLDOVA-BACKUP-SSH-EXECUTOR-RCA-1145R1", + "status": "OK", + "rc": 0, + "host": "pve01", + "mode": "read-only", + "component": "homelab-moldova-backup-ssh-executor-rca", + "started_at_utc": "2026-09-23T22:10:36Z", + "finished_at_utc": "2026-09-23T22:10:37Z", + "reference_register_checked": true, + "reference_sha256": "5a3d8e5154c41cb582a4e0aca68090be1f0138918bf82131a948df326f9d8d66", + "error_register_checked": true, + "error_register_sha256": "3b09a553ec0f527ed3afeed4753f52a74ee3036045bbfb3c685e9f8af4ba7ba0", + "command_sha256": "42ee4c714560544f20bfc87992484e5d4e4ad99ab5eadf2e0334fcfd5cc7e586", + "duplicate_failed_command_blocked": false, + "block_reason": null, + "execution_started": true, + "change_declared": false, + "result_contract_valid": true, + "result_contract_status": null, + "result_contract_error": null, + "command_rc": 0, + "changes_made": false, + "rollback_started": false, + "rollback_restored": null, + "mutation_outcome": "NO_MUTATION", + "sanitized": true, + "secrets_included": false, + "private_addresses_included": false, + "raw_evidence_retained_locally": true, + "raw_evidence_sha256": "06d7265e11af10711ebcbd148af88f0163f786ca78ac3be9eb2a43259e608e4c", + "sanitized_output_sha256": "06d7265e11af10711ebcbd148af88f0163f786ca78ac3be9eb2a43259e608e4c", + "output_truncated_in_json": false, + "full_sanitized_output_url": "https://git.gram1.ru/.well-known/homelab-runtime/latest.txt", + "output": "WORKERS2_MOLDOVA_BACKUP_SSH_EXECUTOR_RCA_BEGIN=1\nCOMMAND_ID=SUPPORT-260924-HOMELAB-MOLDOVA-BACKUP-SSH-EXECUTOR-RCA-1145R1\nMODE=read-only\nCOMPONENT=homelab-moldova-backup-ssh-executor-rca\nMUTATION_BOUNDARY=NONE;PVE01_LOCAL_RELEASE_EXECUTOR_DISCOVERY_ONLY;NO_MOLDOVA_CONNECTION;NO_REMOTE_SSH;NO_PROVIDER_CALL;NO_SYSTEMD_ACTION;NO_BACKUP_RUN;NO_BACKUP_MUTATION;NO_TARGET_POLICY_MUTATION;NO_NETBIRD_MUTATION;NO_MONITORING_MUTATION;NO_GIT_WRITE;NO_SECRET_VALUE_READOUT\nPREDECESSOR_1144=PASS_MOLDOVA_1144_SSH_AUTH_RCA_COLLECTED\nPREDECESSOR_1144_AUTH_TRANSPORT_CLASS=OPENSSH_EFFECTIVE_IDENTITY_FILE_EXISTS\nRCA_REASON=1144_PROVED_DEFAULT_IDENTITIES_EXIST_BUT_DID_NOT_DISCOVER_ACTUAL_RELEASE_EXECUTOR\nCARRIER_BASELINE=KNOWN_GOOD_1137_BYTE_SEMANTICS\nREFERENCE_SHA256=5a3d8e5154c41cb582a4e0aca68090be1f0138918bf82131a948df326f9d8d66\nERROR_REGISTER_SHA256=3b09a553ec0f527ed3afeed4753f52a74ee3036045bbfb3c685e9f8af4ba7ba0\nRUNNER_SHA256=b248a4c32c9cc64e5747e7dce6c7fc0a23f5124a77c71ce72e27a81aceae9d2d\nTARGETS_SHA256=5cba5f1e14c65211c9fab6d11594c19279ab68369b4db3f7283708c368348288\nMD_RELAY_COUNT=1\nMD_RELAY_ENABLED=false\nMD_RELAY_TYPE=vps_md\nMD_RELAY_SCOPE=vps\nWRAPPER_META_BEGIN=1\nWRAPPER_PATH=/usr/local/sbin/homelab-backup-v2\nWRAPPER_SHA256=5077444065c732451cb84898680f62361b21a24ef9eb4f191253e82ead524ea2\nWRAPPER_BYTES=1656\nWRAPPER_MODE=750\nWRAPPER_OWNER=root:root\nWRAPPER_META_END=1\nWRAPPER_DISCOVERY_SNIPPETS_BEGIN=1\nWRAPPER_SRC|line=1|text=#!/usr/bin/python3\nWRAPPER_SRC|line=4|text=ACCEPTED=pathlib.Path('/opt/homelab-backup-v2/releases/backup-v2-overlay-504r1-ca6876fa9c3b')\nWRAPPER_SRC|line=5|text=EXPECTED={'fresh_lxc_helper.py': 'dbb3e6dbb71b10797df736431d9501a752a0f8c439163c9a0bddb7f76747f011', 'producer_exec_helper.py': 'b3b97099bcd7126670f866e81faae6a42a96d869d9989e6a67f6ea6c871c90b3', 'lxc_adapter.py': '3fa7d194010d76f7c8bc95035ae8876262e9337903e310f45f45df88c266dc8d', 'qemu_adapter.py': '1906db42bd34bbe85eb46fd9797b3572d46e0a41c29d1c29be5f9df5e448f15b', 'scheduler_adapter.py': 'a6f67159a64067773323f028126e3109e15cf17a8142cea5e6129970f1d4da96', 'entry504.py': 'ca6876fa9c3b1465b045344a8ad46962de16972c7dadc4ac6257c7d225047793'}\nWRAPPER_SRC|line=7|text= h=hashlib.sha256()\nWRAPPER_SRC|line=16|text= if not stat.S_ISDIR(st.st_mode) or stat.S_ISLNK(st.st_mode) or st.st_uid!=0 or st.st_gid!=0 or stat.S_IMODE(st.st_mode)!=0o500: raise SystemExit('RELEASE_ROOT_IDENTITY')\nWRAPPER_SRC|line=18|text= if names!=sorted(EXPECTED): raise SystemExit('RELEASE_FILESET')\nWRAPPER_SRC|line=21|text= if not stat.S_ISREG(s.st_mode) or stat.S_ISLNK(s.st_mode) or s.st_uid!=0 or s.st_gid!=0 or stat.S_IMODE(s.st_mode)!=0o400 or sha(p)!=want: raise SystemExit('RELEASE_FILE_IDENTITY_'+name)\nWRAPPER_DISCOVERY_SNIPPETS_END=1\nRELEASE_LINKS_BEGIN=1\nRELEASE_PATH|path=/var/lib/homelab-backup-v2|type=directory|mode=700|owner=root:root\nRELEASE_PATH|path=/etc/homelab-backup-v2|type=directory|mode=700|owner=root:root\nRELEASE_LINKS_END=1\nEXECUTOR_MATCH_FILES_BEGIN=1\nEXECUTOR_MATCH_FILE_COUNT=2\nEXECUTOR_FILE|path=/etc/homelab-backup-v2/targets.json|mode=600|owner=root|group=root|bytes=7415|sha256=5cba5f1e14c65211c9fab6d11594c19279ab68369b4db3f7283708c368348288\nEXECUTOR_FILE|path=/var/lib/homelab-backup-v2/state/md-relay.json|mode=600|owner=root|group=root|bytes=615|sha256=24c6e2f71fb84f1f2dcd5e95cd68a527f2c811254f4be3e666985f1a36859f76\nEXECUTOR_MATCH_FILES_END=1\nEXECUTOR_AUTH_SNIPPETS_BEGIN=1\nEXECUTOR_SRC|path=/etc/homelab-backup-v2/targets.json|line=168|text= \"hostkey_fingerprint\": \"SHA256:DXJGwunC5tEVvnC3nRxyayTN8FXmyJcVgf+Oa6udQsY\",\nEXECUTOR_SRC|path=/etc/homelab-backup-v2/targets.json|line=169|text= \"logical_id\": \"md-relay\",\nEXECUTOR_SRC|path=/etc/homelab-backup-v2/targets.json|line=172|text= \"type\": \"vps_md\",\nEXECUTOR_SRC|path=/etc/homelab-backup-v2/targets.json|line=180|text= \"hostkey_fingerprint\": \"SHA256:Ef+HsZb8lIXoadvYBXuGKIL3H7NyJMmf9EYCoJRoSv8\",\nEXECUTOR_SRC|path=/etc/homelab-backup-v2/targets.json|line=192|text= \"hostkey_fingerprint\": \"SHA256:805Q9eh/fePUci42dfY58I6q0lFiiPHn0/M+mckypmE\",\nEXECUTOR_SRC|path=/var/lib/homelab-backup-v2/state/md-relay.json|line=1|text={\"consecutive_failures\":0,\"failure_backoff_seconds\":0,\"generation\":\"v2-md-relay-20260923T003832Z-8051ad720cce\",\"green_verified\":false,\"last_attempt_epoch\":1790123935,\"last_success_epoch\":1790123935,\"logical_id\":\"md-relay\",\"manifest_sha256\":\"b57ca8ee55d8622f6aeb044d3063fd0bb7cb6ada83f50511c87894d083a2a428\",\"payload_sha256\":\"0d65b5ba587ec9f393e7a54af1df2e86b54332509da617d6bfe594b42d5fa3f8\",\"replicas\":[{\"crypt\":\"pve03-mail-01-crypt\",\"physical\":\"pve03-mail-01\"},{\"crypt\":\"pve01-mail-04-crypt\",\"physical\":\"pve01-mail-04\"}],\"result\":\"SUCCESS\",\"retry_after_epoch\":0,\"schema\":1,\"updated_at_utc\":\"2026-09-23T00:38:55Z\"}\nEXECUTOR_AUTH_SNIPPET_COUNT=6\nEXECUTOR_AUTH_SNIPPETS_END=1\nMD_RELAY_STATE_BEGIN=1\nMD_RELAY_STATE_SHA256=24c6e2f71fb84f1f2dcd5e95cd68a527f2c811254f4be3e666985f1a36859f76\nSTATE_LOGICAL_ID=md-relay\nSTATE_RESULT=SUCCESS\nSTATE_GENERATION=v2-md-relay-20260923T003832Z-8051ad720cce\nSTATE_UPDATED_AT_UTC=2026-09-23T00:38:55Z\nSTATE_CONSECUTIVE_FAILURES=0\nSTATE_FAILURE_BACKOFF_SECONDS=0\nSTATE_REPLICA_COUNT=2\nMD_RELAY_STATE_END=1\nMD_RELAY_SUCCESS_WINDOW_BEGIN=1\nSep 23 03:38:32 pve01 systemd[1]: Starting homelab-backup-v2.service - Homelab Backup V2 manifest-driven scheduler...\nSep 23 03:38:32 pve01 homelab-backup-v2[2925325]: SELFTEST_REPLACEMENT486=PASS\nSep 23 03:38:55 pve01 homelab-backup-v2[2925325]: SCHEDULE_TARGET=md-relay\nSep 23 03:38:55 pve01 homelab-backup-v2[2925325]: SCHEDULE_RESULT=SUCCESS\nSep 23 03:38:55 pve01 systemd[1]: homelab-backup-v2.service: Deactivated successfully.\nSep 23 03:38:55 pve01 systemd[1]: Finished homelab-backup-v2.service - Homelab Backup V2 manifest-driven scheduler.\nSep 23 03:38:55 pve01 systemd[1]: homelab-backup-v2.service: Consumed 3.449s CPU time, 120.2M memory peak.\nMD_RELAY_SUCCESS_WINDOW_END=1\nAUTH_EVIDENCE_EXPLICIT_IDENTITY_FILE_COUNT=0\nAUTH_EVIDENCE_STRICT_HOSTKEY_FILE_COUNT=0\nAUTH_EVIDENCE_USER_KNOWN_HOSTS_FILE_COUNT=0\nAUTH_EVIDENCE_BATCHMODE_FILE_COUNT=0\nAUTH_EVIDENCE_OPS_USER_FILE_COUNT=1\nBACKUPV2_MD_RELAY_AUTH_CLASS=EXECUTOR_FOUND_AUTH_OPTIONS_NOT_FULLY_CLASSIFIED\nDECISION=PASS_MOLDOVA_1145_BACKUP_SSH_EXECUTOR_RCA\nNEXT_GATE=READ_EXACT_1145_AND_BUILD_REMOTE_ROLE_PREFLIGHT_WITH_EXACT_EXECUTOR_AUTH\nTASK_RESULT=PASS_MOLDOVA_BACKUP_SSH_EXECUTOR_RCA\nHOMELAB_RESULT_CONTRACT={\"version\":1,\"command_id\":\"SUPPORT-260924-HOMELAB-MOLDOVA-BACKUP-SSH-EXECUTOR-RCA-1145R1\",\"status\":\"OK\",\"changes_made\":false,\"rollback_started\":false,\"rollback_restored\":null}\nWORKERS2_MOLDOVA_BACKUP_SSH_EXECUTOR_RCA_END=1\n" +} diff --git a/runtime/history/SUPPORT-260924-HOMELAB-MOLDOVA-BACKUP-SSH-EXECUTOR-RCA-1145R1.txt b/runtime/history/SUPPORT-260924-HOMELAB-MOLDOVA-BACKUP-SSH-EXECUTOR-RCA-1145R1.txt new file mode 100644 index 00000000..b0d284c1 --- /dev/null +++ b/runtime/history/SUPPORT-260924-HOMELAB-MOLDOVA-BACKUP-SSH-EXECUTOR-RCA-1145R1.txt @@ -0,0 +1,113 @@ +CHAT_OUTPUT_BEGIN +COMMAND_ID=SUPPORT-260924-HOMELAB-MOLDOVA-BACKUP-SSH-EXECUTOR-RCA-1145R1 +STATUS=OK +RC=0 +HOST=pve01 +MODE=read-only +COMPONENT=homelab-moldova-backup-ssh-executor-rca +REFERENCE_REGISTER_CHECK=OK +REFERENCE_SHA256=5a3d8e5154c41cb582a4e0aca68090be1f0138918bf82131a948df326f9d8d66 +ERROR_REGISTER_CHECK=OK +ERROR_REGISTER_SHA256=3b09a553ec0f527ed3afeed4753f52a74ee3036045bbfb3c685e9f8af4ba7ba0 +COMMAND_SHA256=42ee4c714560544f20bfc87992484e5d4e4ad99ab5eadf2e0334fcfd5cc7e586 +DUPLICATE_FAILED_COMMAND_BLOCKED=false +EXECUTION_STARTED=true +CHANGE_DECLARED=false +RESULT_CONTRACT_VALID=true +RESULT_CONTRACT_STATUS=NOT_APPLICABLE +RESULT_CONTRACT_ERROR=NONE +COMMAND_RC=0 +CHANGES_MADE=false +ROLLBACK_STARTED=false +ROLLBACK_RESTORED=null +MUTATION_OUTCOME=NO_MUTATION +SANITIZED=yes +SECRETS_INCLUDED=no +PRIVATE_ADDRESSES_INCLUDED=no +RAW_EVIDENCE_SHA256=06d7265e11af10711ebcbd148af88f0163f786ca78ac3be9eb2a43259e608e4c +SANITIZED_OUTPUT_SHA256=06d7265e11af10711ebcbd148af88f0163f786ca78ac3be9eb2a43259e608e4c +OUTPUT_BEGIN +WORKERS2_MOLDOVA_BACKUP_SSH_EXECUTOR_RCA_BEGIN=1 +COMMAND_ID=SUPPORT-260924-HOMELAB-MOLDOVA-BACKUP-SSH-EXECUTOR-RCA-1145R1 +MODE=read-only +COMPONENT=homelab-moldova-backup-ssh-executor-rca +MUTATION_BOUNDARY=NONE;PVE01_LOCAL_RELEASE_EXECUTOR_DISCOVERY_ONLY;NO_MOLDOVA_CONNECTION;NO_REMOTE_SSH;NO_PROVIDER_CALL;NO_SYSTEMD_ACTION;NO_BACKUP_RUN;NO_BACKUP_MUTATION;NO_TARGET_POLICY_MUTATION;NO_NETBIRD_MUTATION;NO_MONITORING_MUTATION;NO_GIT_WRITE;NO_SECRET_VALUE_READOUT +PREDECESSOR_1144=PASS_MOLDOVA_1144_SSH_AUTH_RCA_COLLECTED +PREDECESSOR_1144_AUTH_TRANSPORT_CLASS=OPENSSH_EFFECTIVE_IDENTITY_FILE_EXISTS +RCA_REASON=1144_PROVED_DEFAULT_IDENTITIES_EXIST_BUT_DID_NOT_DISCOVER_ACTUAL_RELEASE_EXECUTOR +CARRIER_BASELINE=KNOWN_GOOD_1137_BYTE_SEMANTICS +REFERENCE_SHA256=5a3d8e5154c41cb582a4e0aca68090be1f0138918bf82131a948df326f9d8d66 +ERROR_REGISTER_SHA256=3b09a553ec0f527ed3afeed4753f52a74ee3036045bbfb3c685e9f8af4ba7ba0 +RUNNER_SHA256=b248a4c32c9cc64e5747e7dce6c7fc0a23f5124a77c71ce72e27a81aceae9d2d +TARGETS_SHA256=5cba5f1e14c65211c9fab6d11594c19279ab68369b4db3f7283708c368348288 +MD_RELAY_COUNT=1 +MD_RELAY_ENABLED=false +MD_RELAY_TYPE=vps_md +MD_RELAY_SCOPE=vps +WRAPPER_META_BEGIN=1 +WRAPPER_PATH=/usr/local/sbin/homelab-backup-v2 +WRAPPER_SHA256=5077444065c732451cb84898680f62361b21a24ef9eb4f191253e82ead524ea2 +WRAPPER_BYTES=1656 +WRAPPER_MODE=750 +WRAPPER_OWNER=root:root +WRAPPER_META_END=1 +WRAPPER_DISCOVERY_SNIPPETS_BEGIN=1 +WRAPPER_SRC|line=1|text=#!/usr/bin/python3 +WRAPPER_SRC|line=4|text=ACCEPTED=pathlib.Path('/opt/homelab-backup-v2/releases/backup-v2-overlay-504r1-ca6876fa9c3b') +WRAPPER_SRC|line=5|text=EXPECTED={'fresh_lxc_helper.py': 'dbb3e6dbb71b10797df736431d9501a752a0f8c439163c9a0bddb7f76747f011', 'producer_exec_helper.py': 'b3b97099bcd7126670f866e81faae6a42a96d869d9989e6a67f6ea6c871c90b3', 'lxc_adapter.py': '3fa7d194010d76f7c8bc95035ae8876262e9337903e310f45f45df88c266dc8d', 'qemu_adapter.py': '1906db42bd34bbe85eb46fd9797b3572d46e0a41c29d1c29be5f9df5e448f15b', 'scheduler_adapter.py': 'a6f67159a64067773323f028126e3109e15cf17a8142cea5e6129970f1d4da96', 'entry504.py': 'ca6876fa9c3b1465b045344a8ad46962de16972c7dadc4ac6257c7d225047793'} +WRAPPER_SRC|line=7|text= h=hashlib.sha256() +WRAPPER_SRC|line=16|text= if not stat.S_ISDIR(st.st_mode) or stat.S_ISLNK(st.st_mode) or st.st_uid!=0 or st.st_gid!=0 or stat.S_IMODE(st.st_mode)!=0o500: raise SystemExit('RELEASE_ROOT_IDENTITY') +WRAPPER_SRC|line=18|text= if names!=sorted(EXPECTED): raise SystemExit('RELEASE_FILESET') +WRAPPER_SRC|line=21|text= if not stat.S_ISREG(s.st_mode) or stat.S_ISLNK(s.st_mode) or s.st_uid!=0 or s.st_gid!=0 or stat.S_IMODE(s.st_mode)!=0o400 or sha(p)!=want: raise SystemExit('RELEASE_FILE_IDENTITY_'+name) +WRAPPER_DISCOVERY_SNIPPETS_END=1 +RELEASE_LINKS_BEGIN=1 +RELEASE_PATH|path=/var/lib/homelab-backup-v2|type=directory|mode=700|owner=root:root +RELEASE_PATH|path=/etc/homelab-backup-v2|type=directory|mode=700|owner=root:root +RELEASE_LINKS_END=1 +EXECUTOR_MATCH_FILES_BEGIN=1 +EXECUTOR_MATCH_FILE_COUNT=2 +EXECUTOR_FILE|path=/etc/homelab-backup-v2/targets.json|mode=600|owner=root|group=root|bytes=7415|sha256=5cba5f1e14c65211c9fab6d11594c19279ab68369b4db3f7283708c368348288 +EXECUTOR_FILE|path=/var/lib/homelab-backup-v2/state/md-relay.json|mode=600|owner=root|group=root|bytes=615|sha256=24c6e2f71fb84f1f2dcd5e95cd68a527f2c811254f4be3e666985f1a36859f76 +EXECUTOR_MATCH_FILES_END=1 +EXECUTOR_AUTH_SNIPPETS_BEGIN=1 +EXECUTOR_SRC|path=/etc/homelab-backup-v2/targets.json|line=168|text= "hostkey_fingerprint": "SHA256:DXJGwunC5tEVvnC3nRxyayTN8FXmyJcVgf+Oa6udQsY", +EXECUTOR_SRC|path=/etc/homelab-backup-v2/targets.json|line=169|text= "logical_id": "md-relay", +EXECUTOR_SRC|path=/etc/homelab-backup-v2/targets.json|line=172|text= "type": "vps_md", +EXECUTOR_SRC|path=/etc/homelab-backup-v2/targets.json|line=180|text= "hostkey_fingerprint": "SHA256:Ef+HsZb8lIXoadvYBXuGKIL3H7NyJMmf9EYCoJRoSv8", +EXECUTOR_SRC|path=/etc/homelab-backup-v2/targets.json|line=192|text= "hostkey_fingerprint": "SHA256:805Q9eh/fePUci42dfY58I6q0lFiiPHn0/M+mckypmE", +EXECUTOR_SRC|path=/var/lib/homelab-backup-v2/state/md-relay.json|line=1|text={"consecutive_failures":0,"failure_backoff_seconds":0,"generation":"v2-md-relay-20260923T003832Z-8051ad720cce","green_verified":false,"last_attempt_epoch":1790123935,"last_success_epoch":1790123935,"logical_id":"md-relay","manifest_sha256":"b57ca8ee55d8622f6aeb044d3063fd0bb7cb6ada83f50511c87894d083a2a428","payload_sha256":"0d65b5ba587ec9f393e7a54af1df2e86b54332509da617d6bfe594b42d5fa3f8","replicas":[{"crypt":"pve03-mail-01-crypt","physical":"pve03-mail-01"},{"crypt":"pve01-mail-04-crypt","physical":"pve01-mail-04"}],"result":"SUCCESS","retry_after_epoch":0,"schema":1,"updated_at_utc":"2026-09-23T00:38:55Z"} +EXECUTOR_AUTH_SNIPPET_COUNT=6 +EXECUTOR_AUTH_SNIPPETS_END=1 +MD_RELAY_STATE_BEGIN=1 +MD_RELAY_STATE_SHA256=24c6e2f71fb84f1f2dcd5e95cd68a527f2c811254f4be3e666985f1a36859f76 +STATE_LOGICAL_ID=md-relay +STATE_RESULT=SUCCESS +STATE_GENERATION=v2-md-relay-20260923T003832Z-8051ad720cce +STATE_UPDATED_AT_UTC=2026-09-23T00:38:55Z +STATE_CONSECUTIVE_FAILURES=0 +STATE_FAILURE_BACKOFF_SECONDS=0 +STATE_REPLICA_COUNT=2 +MD_RELAY_STATE_END=1 +MD_RELAY_SUCCESS_WINDOW_BEGIN=1 +Sep 23 03:38:32 pve01 systemd[1]: Starting homelab-backup-v2.service - Homelab Backup V2 manifest-driven scheduler... +Sep 23 03:38:32 pve01 homelab-backup-v2[2925325]: SELFTEST_REPLACEMENT486=PASS +Sep 23 03:38:55 pve01 homelab-backup-v2[2925325]: SCHEDULE_TARGET=md-relay +Sep 23 03:38:55 pve01 homelab-backup-v2[2925325]: SCHEDULE_RESULT=SUCCESS +Sep 23 03:38:55 pve01 systemd[1]: homelab-backup-v2.service: Deactivated successfully. +Sep 23 03:38:55 pve01 systemd[1]: Finished homelab-backup-v2.service - Homelab Backup V2 manifest-driven scheduler. +Sep 23 03:38:55 pve01 systemd[1]: homelab-backup-v2.service: Consumed 3.449s CPU time, 120.2M memory peak. +MD_RELAY_SUCCESS_WINDOW_END=1 +AUTH_EVIDENCE_EXPLICIT_IDENTITY_FILE_COUNT=0 +AUTH_EVIDENCE_STRICT_HOSTKEY_FILE_COUNT=0 +AUTH_EVIDENCE_USER_KNOWN_HOSTS_FILE_COUNT=0 +AUTH_EVIDENCE_BATCHMODE_FILE_COUNT=0 +AUTH_EVIDENCE_OPS_USER_FILE_COUNT=1 +BACKUPV2_MD_RELAY_AUTH_CLASS=EXECUTOR_FOUND_AUTH_OPTIONS_NOT_FULLY_CLASSIFIED +DECISION=PASS_MOLDOVA_1145_BACKUP_SSH_EXECUTOR_RCA +NEXT_GATE=READ_EXACT_1145_AND_BUILD_REMOTE_ROLE_PREFLIGHT_WITH_EXACT_EXECUTOR_AUTH +TASK_RESULT=PASS_MOLDOVA_BACKUP_SSH_EXECUTOR_RCA +HOMELAB_RESULT_CONTRACT={"version":1,"command_id":"SUPPORT-260924-HOMELAB-MOLDOVA-BACKUP-SSH-EXECUTOR-RCA-1145R1","status":"OK","changes_made":false,"rollback_started":false,"rollback_restored":null} +WORKERS2_MOLDOVA_BACKUP_SSH_EXECUTOR_RCA_END=1 + +OUTPUT_END +CHAT_OUTPUT_END diff --git a/runtime/latest.json b/runtime/latest.json index d2821f3a..e5b31ac8 100644 --- a/runtime/latest.json +++ b/runtime/latest.json @@ -1,38 +1,38 @@ { "schema_version": 1, "channel": "homelab-runtime", - "command_id": "FORUM-260924-EXPAND-8FORUM-ALERTS4A27R1", - "status": "FAIL", - "rc": 3, + "command_id": "SUPPORT-260924-HOMELAB-MOLDOVA-BACKUP-SSH-EXECUTOR-RCA-1145R1", + "status": "OK", + "rc": 0, "host": "pve01", - "mode": "change", - "component": "forum-eight-fleet-alert-expansion", - "started_at_utc": "2026-09-23T22:08:18Z", - "finished_at_utc": "2026-09-23T22:08:35Z", + "mode": "read-only", + "component": "homelab-moldova-backup-ssh-executor-rca", + "started_at_utc": "2026-09-23T22:10:36Z", + "finished_at_utc": "2026-09-23T22:10:37Z", "reference_register_checked": true, "reference_sha256": "5a3d8e5154c41cb582a4e0aca68090be1f0138918bf82131a948df326f9d8d66", "error_register_checked": true, "error_register_sha256": "3b09a553ec0f527ed3afeed4753f52a74ee3036045bbfb3c685e9f8af4ba7ba0", - "command_sha256": "fb2e174911ecb4ef134dedbefdb06441c0aa16d8d21396392b08c117fd9818d3", + "command_sha256": "42ee4c714560544f20bfc87992484e5d4e4ad99ab5eadf2e0334fcfd5cc7e586", "duplicate_failed_command_blocked": false, "block_reason": null, "execution_started": true, - "change_declared": true, + "change_declared": false, "result_contract_valid": true, - "result_contract_status": "FAIL", + "result_contract_status": null, "result_contract_error": null, - "command_rc": 3, - "changes_made": true, - "rollback_started": true, - "rollback_restored": true, - "mutation_outcome": "ROLLED_BACK", + "command_rc": 0, + "changes_made": false, + "rollback_started": false, + "rollback_restored": null, + "mutation_outcome": "NO_MUTATION", "sanitized": true, "secrets_included": false, "private_addresses_included": false, "raw_evidence_retained_locally": true, - "raw_evidence_sha256": "f67b497be09315cbc8b9ab9f46e25c60269ebcce62bd3fdf3dd1a681ad7f7fd5", - "sanitized_output_sha256": "f67b497be09315cbc8b9ab9f46e25c60269ebcce62bd3fdf3dd1a681ad7f7fd5", + "raw_evidence_sha256": "06d7265e11af10711ebcbd148af88f0163f786ca78ac3be9eb2a43259e608e4c", + "sanitized_output_sha256": "06d7265e11af10711ebcbd148af88f0163f786ca78ac3be9eb2a43259e608e4c", "output_truncated_in_json": false, "full_sanitized_output_url": "https://git.gram1.ru/.well-known/homelab-runtime/latest.txt", - "output": "CID=FORUM-260924-EXPAND-8FORUM-ALERTS4A27R1\nMODE=change\nCOMPONENT=forum-eight-fleet-alert-expansion\nMUTATION_BOUNDARY=PVE01_FORUM_PUBLIC_SMOKE_6_TO_8;VM160_WATCHER_FLEET_TEXT_ONLY;VM100_NEW_ERROR_WATCHER;VM210_NEW_ERROR_WATCHER;TWO_RUSSIAN_CANARY_EMAILS;NO_XF_DB_WRITE;NO_XF_ERROR_DELETE;NO_MSMTP_CONFIG_CHANGE;NO_NGINX_PHP_MARIADB_RESTART;NO_VM_CONFIG;NO_CLOUD\nEXECUTION_STARTED=true\nCHANGES_MADE=false\nROLLBACK_STARTED=false\nROLLBACK_RESTORED=null\nALERT_RECIPIENT_SHA256=0d4cf60ca8cd53a12ad50dc5419d3c9c32192f0852000eb10051fbdef52b04bf\nHEALTH_PRE_SHA256=4ee0c432fd906ba2d73198c965d0e57e4e7001781d61161b64f3c51625f034bd\n=== VM100 PREFLIGHT ===\nVM100_STATE=running\nVM100_QGA_TRANSPORT_RC=0\nGUEST_EXITCODE=0\nROOTS=1\nDOMAINS=pvepro.ru\nFLEET_DOMAINS=codevipe.ru,dsmods.ru,gamevipe.ru,hkmods.ru,kingofwolk.ru,zakrutim.ru,pvepro.ru,newfi.ru\nSOURCE_LABEL=pvepro.ru\nALERT_RECIPIENT_SHA256=0d4cf60ca8cd53a12ad50dc5419d3c9c32192f0852000eb10051fbdef52b04bf\nCONTACT_RECIPIENT_HASH_SET_COUNT=1\nPREFLIGHT=PASS\nALERT_RECIPIENT_SHA256=0d4cf60ca8cd53a12ad50dc5419d3c9c32192f0852000eb10051fbdef52b04bf\nGUEST_PREFLIGHT=PASS\nGUEST_STDERR_PRESENT=false\n=== VM210 PREFLIGHT ===\nVM210_STATE=running\nVM210_QGA_TRANSPORT_RC=0\nGUEST_EXITCODE=0\nROOTS=1\nDOMAINS=newfi.ru\nFLEET_DOMAINS=codevipe.ru,dsmods.ru,gamevipe.ru,hkmods.ru,kingofwolk.ru,zakrutim.ru,pvepro.ru,newfi.ru\nSOURCE_LABEL=newfi.ru\nALERT_RECIPIENT_SHA256=0d4cf60ca8cd53a12ad50dc5419d3c9c32192f0852000eb10051fbdef52b04bf\nCONTACT_RECIPIENT_HASH_SET_COUNT=1\nPREFLIGHT=PASS\nALERT_RECIPIENT_SHA256=0d4cf60ca8cd53a12ad50dc5419d3c9c32192f0852000eb10051fbdef52b04bf\nGUEST_PREFLIGHT=PASS\nGUEST_STDERR_PRESENT=false\n=== VM160 PREFLIGHT ===\nVM160_STATE=running\nVM160_QGA_TRANSPORT_RC=0\nVM160_GUEST_EXITCODE=0\nVM160_MONITOR_PRE_SHA256=5d3055077aafb084c0f6424dcd33950f589296dc21a52024e7b79adc7e9c2dd1\nROOTS=6\nDOMAINS=codevipe.ru,dsmods.ru,gamevipe.ru,hkmods.ru,kingofwolk.ru,zakrutim.ru\nFLEET_DOMAINS=codevipe.ru,dsmods.ru,gamevipe.ru,hkmods.ru,kingofwolk.ru,zakrutim.ru,pvepro.ru,newfi.ru\nSOURCE_LABEL=forum-prod\nALERT_RECIPIENT_SHA256=0d4cf60ca8cd53a12ad50dc5419d3c9c32192f0852000eb10051fbdef52b04bf\nCONTACT_RECIPIENT_HASH_SET_COUNT=1\nPREFLIGHT=PASS\nVM160_PREFLIGHT=PASS\nVM160_GUEST_STDERR_PRESENT=false\nCHANGES_MADE=true\nHEALTH_POST_SHA256=5900ae2822bcb5abd7ec3c380f2e6d9a16658f0d7f5e3f8cc9ccbd62764b5337\nHEALTH_RUN_RC=0\nHEALTH_LINE=STATUS=OK TS=2026-09-23T22:08:25Z TYPE=forum-snuffleupagus BAD=0 WARN=0 MODE=low-noise-simulation STRICT=NO MODULE=loaded FORUMS=8 PUBLIC_SMOKE_BAD=0 ACTUAL_READONLY_EXEC_RULE=NO REPORT=/var/lib/homelab-forum-snuffleupagus/latest.md\n=== VM160 APPLY ===\nVM160_STATE=running\nVM160_QGA_TRANSPORT_RC=0\nVM160_GUEST_EXITCODE=0\nVM160_MONITOR_PRE_SHA256=5d3055077aafb084c0f6424dcd33950f589296dc21a52024e7b79adc7e9c2dd1\nROOTS=6\nDOMAINS=codevipe.ru,dsmods.ru,gamevipe.ru,hkmods.ru,kingofwolk.ru,zakrutim.ru\nFLEET_DOMAINS=codevipe.ru,dsmods.ru,gamevipe.ru,hkmods.ru,kingofwolk.ru,zakrutim.ru,pvepro.ru,newfi.ru\nSOURCE_LABEL=forum-prod\nALERT_RECIPIENT_SHA256=0d4cf60ca8cd53a12ad50dc5419d3c9c32192f0852000eb10051fbdef52b04bf\nCONTACT_RECIPIENT_HASH_SET_COUNT=1\nPREFLIGHT=PASS\nVM160_PREFLIGHT=PASS\nVM160_MONITOR_POST_SHA256=3417dc26c8292ac3b07be8aafb27bd5228d5876bda61bc1a0a38be4e85ea7052\nVM160_ALERT_TIMER_ACTIVE=active\nVM160_ALERT_TIMER_ENABLED=enabled\nVM160_APPLY=PASS\nVM160_GUEST_STDERR_PRESENT=false\n=== VM100 APPLY/CANARY ===\nVM100_STATE=running\nVM100_QGA_TRANSPORT_RC=0\nGUEST_EXITCODE=90\nROOTS=1\nDOMAINS=pvepro.ru\nFLEET_DOMAINS=codevipe.ru,dsmods.ru,gamevipe.ru,hkmods.ru,kingofwolk.ru,zakrutim.ru,pvepro.ru,newfi.ru\nSOURCE_LABEL=pvepro.ru\nALERT_RECIPIENT_SHA256=0d4cf60ca8cd53a12ad50dc5419d3c9c32192f0852000eb10051fbdef52b04bf\nCONTACT_RECIPIENT_HASH_SET_COUNT=1\nPREFLIGHT=PASS\nALERT_RECIPIENT_SHA256=0d4cf60ca8cd53a12ad50dc5419d3c9c32192f0852000eb10051fbdef52b04bf\nGUEST_PREFLIGHT=PASS\nROOTS=1\nDOMAINS=pvepro.ru\nFLEET_DOMAINS=codevipe.ru,dsmods.ru,gamevipe.ru,hkmods.ru,kingofwolk.ru,zakrutim.ru,pvepro.ru,newfi.ru\nSOURCE_LABEL=pvepro.ru\nALERT_RECIPIENT_SHA256=0d4cf60ca8cd53a12ad50dc5419d3c9c32192f0852000eb10051fbdef52b04bf\nCONTACT_RECIPIENT_HASH_SET_COUNT=1\nINITIALIZE=PASS\nALERT_TIMER_ACTIVE=active\nALERT_TIMER_ENABLED=enabled\nSTATE_SITE_COUNT=1\nCANARY_BEGIN=true\nROOTS=1\nDOMAINS=pvepro.ru\nFLEET_DOMAINS=codevipe.ru,dsmods.ru,gamevipe.ru,hkmods.ru,kingofwolk.ru,zakrutim.ru,pvepro.ru,newfi.ru\nSOURCE_LABEL=pvepro.ru\nALERT_RECIPIENT_SHA256=0d4cf60ca8cd53a12ad50dc5419d3c9c32192f0852000eb10051fbdef52b04bf\nCONTACT_RECIPIENT_HASH_SET_COUNT=1\nTEST_SEND_RC=65\nTEST_SEND_ERR=sendmail: envelope from address not accepted by the server sendmail: server message: 553 5.7.1 Sender address rejected: user not found. 1790201313-X8lFAQ7fpW20-VJHZAQ43 sendmail: could not send mail (account default from /etc/msmtprc)\nALERT_FAILURE=TEST_SEND_FAILED_65\nGUEST_FAILURE=CANARY_SEND\nGUEST_ROLLBACK_STARTED=true\nGUEST_ROLLBACK_RESTORED=true\nGUEST_STDERR_PRESENT=true\nGUEST_STDERR_SHA256=f704b93c5de99a394103c9ff88a07ffd86f97675d45caeb3a7829b72b3152eec\nGUEST_STDERR_BYTES=154\nFAILURE=VM100_APPLY_RC_90\nROLLBACK_STARTED=true\nROLLBACK_RESTORED=true\nDECISION=FAIL_EIGHT_FORUM_EXPANSION_ROLLED_BACK\nNEXT_GATE=REVIEW_FAILURE\nTASK_RESULT=FAIL_EIGHT_FORUM_EXPANSION\nHOMELAB_RESULT_CONTRACT={\"version\":1,\"command_id\":\"FORUM-260924-EXPAND-8FORUM-ALERTS4A27R1\",\"status\":\"FAIL\",\"changes_made\":true,\"rollback_started\":true,\"rollback_restored\":true}\n" + "output": "WORKERS2_MOLDOVA_BACKUP_SSH_EXECUTOR_RCA_BEGIN=1\nCOMMAND_ID=SUPPORT-260924-HOMELAB-MOLDOVA-BACKUP-SSH-EXECUTOR-RCA-1145R1\nMODE=read-only\nCOMPONENT=homelab-moldova-backup-ssh-executor-rca\nMUTATION_BOUNDARY=NONE;PVE01_LOCAL_RELEASE_EXECUTOR_DISCOVERY_ONLY;NO_MOLDOVA_CONNECTION;NO_REMOTE_SSH;NO_PROVIDER_CALL;NO_SYSTEMD_ACTION;NO_BACKUP_RUN;NO_BACKUP_MUTATION;NO_TARGET_POLICY_MUTATION;NO_NETBIRD_MUTATION;NO_MONITORING_MUTATION;NO_GIT_WRITE;NO_SECRET_VALUE_READOUT\nPREDECESSOR_1144=PASS_MOLDOVA_1144_SSH_AUTH_RCA_COLLECTED\nPREDECESSOR_1144_AUTH_TRANSPORT_CLASS=OPENSSH_EFFECTIVE_IDENTITY_FILE_EXISTS\nRCA_REASON=1144_PROVED_DEFAULT_IDENTITIES_EXIST_BUT_DID_NOT_DISCOVER_ACTUAL_RELEASE_EXECUTOR\nCARRIER_BASELINE=KNOWN_GOOD_1137_BYTE_SEMANTICS\nREFERENCE_SHA256=5a3d8e5154c41cb582a4e0aca68090be1f0138918bf82131a948df326f9d8d66\nERROR_REGISTER_SHA256=3b09a553ec0f527ed3afeed4753f52a74ee3036045bbfb3c685e9f8af4ba7ba0\nRUNNER_SHA256=b248a4c32c9cc64e5747e7dce6c7fc0a23f5124a77c71ce72e27a81aceae9d2d\nTARGETS_SHA256=5cba5f1e14c65211c9fab6d11594c19279ab68369b4db3f7283708c368348288\nMD_RELAY_COUNT=1\nMD_RELAY_ENABLED=false\nMD_RELAY_TYPE=vps_md\nMD_RELAY_SCOPE=vps\nWRAPPER_META_BEGIN=1\nWRAPPER_PATH=/usr/local/sbin/homelab-backup-v2\nWRAPPER_SHA256=5077444065c732451cb84898680f62361b21a24ef9eb4f191253e82ead524ea2\nWRAPPER_BYTES=1656\nWRAPPER_MODE=750\nWRAPPER_OWNER=root:root\nWRAPPER_META_END=1\nWRAPPER_DISCOVERY_SNIPPETS_BEGIN=1\nWRAPPER_SRC|line=1|text=#!/usr/bin/python3\nWRAPPER_SRC|line=4|text=ACCEPTED=pathlib.Path('/opt/homelab-backup-v2/releases/backup-v2-overlay-504r1-ca6876fa9c3b')\nWRAPPER_SRC|line=5|text=EXPECTED={'fresh_lxc_helper.py': 'dbb3e6dbb71b10797df736431d9501a752a0f8c439163c9a0bddb7f76747f011', 'producer_exec_helper.py': 'b3b97099bcd7126670f866e81faae6a42a96d869d9989e6a67f6ea6c871c90b3', 'lxc_adapter.py': '3fa7d194010d76f7c8bc95035ae8876262e9337903e310f45f45df88c266dc8d', 'qemu_adapter.py': '1906db42bd34bbe85eb46fd9797b3572d46e0a41c29d1c29be5f9df5e448f15b', 'scheduler_adapter.py': 'a6f67159a64067773323f028126e3109e15cf17a8142cea5e6129970f1d4da96', 'entry504.py': 'ca6876fa9c3b1465b045344a8ad46962de16972c7dadc4ac6257c7d225047793'}\nWRAPPER_SRC|line=7|text= h=hashlib.sha256()\nWRAPPER_SRC|line=16|text= if not stat.S_ISDIR(st.st_mode) or stat.S_ISLNK(st.st_mode) or st.st_uid!=0 or st.st_gid!=0 or stat.S_IMODE(st.st_mode)!=0o500: raise SystemExit('RELEASE_ROOT_IDENTITY')\nWRAPPER_SRC|line=18|text= if names!=sorted(EXPECTED): raise SystemExit('RELEASE_FILESET')\nWRAPPER_SRC|line=21|text= if not stat.S_ISREG(s.st_mode) or stat.S_ISLNK(s.st_mode) or s.st_uid!=0 or s.st_gid!=0 or stat.S_IMODE(s.st_mode)!=0o400 or sha(p)!=want: raise SystemExit('RELEASE_FILE_IDENTITY_'+name)\nWRAPPER_DISCOVERY_SNIPPETS_END=1\nRELEASE_LINKS_BEGIN=1\nRELEASE_PATH|path=/var/lib/homelab-backup-v2|type=directory|mode=700|owner=root:root\nRELEASE_PATH|path=/etc/homelab-backup-v2|type=directory|mode=700|owner=root:root\nRELEASE_LINKS_END=1\nEXECUTOR_MATCH_FILES_BEGIN=1\nEXECUTOR_MATCH_FILE_COUNT=2\nEXECUTOR_FILE|path=/etc/homelab-backup-v2/targets.json|mode=600|owner=root|group=root|bytes=7415|sha256=5cba5f1e14c65211c9fab6d11594c19279ab68369b4db3f7283708c368348288\nEXECUTOR_FILE|path=/var/lib/homelab-backup-v2/state/md-relay.json|mode=600|owner=root|group=root|bytes=615|sha256=24c6e2f71fb84f1f2dcd5e95cd68a527f2c811254f4be3e666985f1a36859f76\nEXECUTOR_MATCH_FILES_END=1\nEXECUTOR_AUTH_SNIPPETS_BEGIN=1\nEXECUTOR_SRC|path=/etc/homelab-backup-v2/targets.json|line=168|text= \"hostkey_fingerprint\": \"SHA256:DXJGwunC5tEVvnC3nRxyayTN8FXmyJcVgf+Oa6udQsY\",\nEXECUTOR_SRC|path=/etc/homelab-backup-v2/targets.json|line=169|text= \"logical_id\": \"md-relay\",\nEXECUTOR_SRC|path=/etc/homelab-backup-v2/targets.json|line=172|text= \"type\": \"vps_md\",\nEXECUTOR_SRC|path=/etc/homelab-backup-v2/targets.json|line=180|text= \"hostkey_fingerprint\": \"SHA256:Ef+HsZb8lIXoadvYBXuGKIL3H7NyJMmf9EYCoJRoSv8\",\nEXECUTOR_SRC|path=/etc/homelab-backup-v2/targets.json|line=192|text= \"hostkey_fingerprint\": \"SHA256:805Q9eh/fePUci42dfY58I6q0lFiiPHn0/M+mckypmE\",\nEXECUTOR_SRC|path=/var/lib/homelab-backup-v2/state/md-relay.json|line=1|text={\"consecutive_failures\":0,\"failure_backoff_seconds\":0,\"generation\":\"v2-md-relay-20260923T003832Z-8051ad720cce\",\"green_verified\":false,\"last_attempt_epoch\":1790123935,\"last_success_epoch\":1790123935,\"logical_id\":\"md-relay\",\"manifest_sha256\":\"b57ca8ee55d8622f6aeb044d3063fd0bb7cb6ada83f50511c87894d083a2a428\",\"payload_sha256\":\"0d65b5ba587ec9f393e7a54af1df2e86b54332509da617d6bfe594b42d5fa3f8\",\"replicas\":[{\"crypt\":\"pve03-mail-01-crypt\",\"physical\":\"pve03-mail-01\"},{\"crypt\":\"pve01-mail-04-crypt\",\"physical\":\"pve01-mail-04\"}],\"result\":\"SUCCESS\",\"retry_after_epoch\":0,\"schema\":1,\"updated_at_utc\":\"2026-09-23T00:38:55Z\"}\nEXECUTOR_AUTH_SNIPPET_COUNT=6\nEXECUTOR_AUTH_SNIPPETS_END=1\nMD_RELAY_STATE_BEGIN=1\nMD_RELAY_STATE_SHA256=24c6e2f71fb84f1f2dcd5e95cd68a527f2c811254f4be3e666985f1a36859f76\nSTATE_LOGICAL_ID=md-relay\nSTATE_RESULT=SUCCESS\nSTATE_GENERATION=v2-md-relay-20260923T003832Z-8051ad720cce\nSTATE_UPDATED_AT_UTC=2026-09-23T00:38:55Z\nSTATE_CONSECUTIVE_FAILURES=0\nSTATE_FAILURE_BACKOFF_SECONDS=0\nSTATE_REPLICA_COUNT=2\nMD_RELAY_STATE_END=1\nMD_RELAY_SUCCESS_WINDOW_BEGIN=1\nSep 23 03:38:32 pve01 systemd[1]: Starting homelab-backup-v2.service - Homelab Backup V2 manifest-driven scheduler...\nSep 23 03:38:32 pve01 homelab-backup-v2[2925325]: SELFTEST_REPLACEMENT486=PASS\nSep 23 03:38:55 pve01 homelab-backup-v2[2925325]: SCHEDULE_TARGET=md-relay\nSep 23 03:38:55 pve01 homelab-backup-v2[2925325]: SCHEDULE_RESULT=SUCCESS\nSep 23 03:38:55 pve01 systemd[1]: homelab-backup-v2.service: Deactivated successfully.\nSep 23 03:38:55 pve01 systemd[1]: Finished homelab-backup-v2.service - Homelab Backup V2 manifest-driven scheduler.\nSep 23 03:38:55 pve01 systemd[1]: homelab-backup-v2.service: Consumed 3.449s CPU time, 120.2M memory peak.\nMD_RELAY_SUCCESS_WINDOW_END=1\nAUTH_EVIDENCE_EXPLICIT_IDENTITY_FILE_COUNT=0\nAUTH_EVIDENCE_STRICT_HOSTKEY_FILE_COUNT=0\nAUTH_EVIDENCE_USER_KNOWN_HOSTS_FILE_COUNT=0\nAUTH_EVIDENCE_BATCHMODE_FILE_COUNT=0\nAUTH_EVIDENCE_OPS_USER_FILE_COUNT=1\nBACKUPV2_MD_RELAY_AUTH_CLASS=EXECUTOR_FOUND_AUTH_OPTIONS_NOT_FULLY_CLASSIFIED\nDECISION=PASS_MOLDOVA_1145_BACKUP_SSH_EXECUTOR_RCA\nNEXT_GATE=READ_EXACT_1145_AND_BUILD_REMOTE_ROLE_PREFLIGHT_WITH_EXACT_EXECUTOR_AUTH\nTASK_RESULT=PASS_MOLDOVA_BACKUP_SSH_EXECUTOR_RCA\nHOMELAB_RESULT_CONTRACT={\"version\":1,\"command_id\":\"SUPPORT-260924-HOMELAB-MOLDOVA-BACKUP-SSH-EXECUTOR-RCA-1145R1\",\"status\":\"OK\",\"changes_made\":false,\"rollback_started\":false,\"rollback_restored\":null}\nWORKERS2_MOLDOVA_BACKUP_SSH_EXECUTOR_RCA_END=1\n" } diff --git a/runtime/latest.txt b/runtime/latest.txt index 0b5149fb..b0d284c1 100644 --- a/runtime/latest.txt +++ b/runtime/latest.txt @@ -1,152 +1,113 @@ CHAT_OUTPUT_BEGIN -COMMAND_ID=FORUM-260924-EXPAND-8FORUM-ALERTS4A27R1 -STATUS=FAIL -RC=3 +COMMAND_ID=SUPPORT-260924-HOMELAB-MOLDOVA-BACKUP-SSH-EXECUTOR-RCA-1145R1 +STATUS=OK +RC=0 HOST=pve01 -MODE=change -COMPONENT=forum-eight-fleet-alert-expansion +MODE=read-only +COMPONENT=homelab-moldova-backup-ssh-executor-rca REFERENCE_REGISTER_CHECK=OK REFERENCE_SHA256=5a3d8e5154c41cb582a4e0aca68090be1f0138918bf82131a948df326f9d8d66 ERROR_REGISTER_CHECK=OK ERROR_REGISTER_SHA256=3b09a553ec0f527ed3afeed4753f52a74ee3036045bbfb3c685e9f8af4ba7ba0 -COMMAND_SHA256=fb2e174911ecb4ef134dedbefdb06441c0aa16d8d21396392b08c117fd9818d3 +COMMAND_SHA256=42ee4c714560544f20bfc87992484e5d4e4ad99ab5eadf2e0334fcfd5cc7e586 DUPLICATE_FAILED_COMMAND_BLOCKED=false EXECUTION_STARTED=true -CHANGE_DECLARED=true +CHANGE_DECLARED=false RESULT_CONTRACT_VALID=true -RESULT_CONTRACT_STATUS=FAIL +RESULT_CONTRACT_STATUS=NOT_APPLICABLE RESULT_CONTRACT_ERROR=NONE -COMMAND_RC=3 -CHANGES_MADE=true -ROLLBACK_STARTED=true -ROLLBACK_RESTORED=true -MUTATION_OUTCOME=ROLLED_BACK -SANITIZED=yes -SECRETS_INCLUDED=no -PRIVATE_ADDRESSES_INCLUDED=no -RAW_EVIDENCE_SHA256=f67b497be09315cbc8b9ab9f46e25c60269ebcce62bd3fdf3dd1a681ad7f7fd5 -SANITIZED_OUTPUT_SHA256=f67b497be09315cbc8b9ab9f46e25c60269ebcce62bd3fdf3dd1a681ad7f7fd5 -OUTPUT_BEGIN -CID=FORUM-260924-EXPAND-8FORUM-ALERTS4A27R1 -MODE=change -COMPONENT=forum-eight-fleet-alert-expansion -MUTATION_BOUNDARY=PVE01_FORUM_PUBLIC_SMOKE_6_TO_8;VM160_WATCHER_FLEET_TEXT_ONLY;VM100_NEW_ERROR_WATCHER;VM210_NEW_ERROR_WATCHER;TWO_RUSSIAN_CANARY_EMAILS;NO_XF_DB_WRITE;NO_XF_ERROR_DELETE;NO_MSMTP_CONFIG_CHANGE;NO_NGINX_PHP_MARIADB_RESTART;NO_VM_CONFIG;NO_CLOUD -EXECUTION_STARTED=true +COMMAND_RC=0 CHANGES_MADE=false ROLLBACK_STARTED=false ROLLBACK_RESTORED=null -ALERT_RECIPIENT_SHA256=0d4cf60ca8cd53a12ad50dc5419d3c9c32192f0852000eb10051fbdef52b04bf -HEALTH_PRE_SHA256=4ee0c432fd906ba2d73198c965d0e57e4e7001781d61161b64f3c51625f034bd -=== VM100 PREFLIGHT === -VM100_STATE=running -VM100_QGA_TRANSPORT_RC=0 -GUEST_EXITCODE=0 -ROOTS=1 -DOMAINS=pvepro.ru -FLEET_DOMAINS=codevipe.ru,dsmods.ru,gamevipe.ru,hkmods.ru,kingofwolk.ru,zakrutim.ru,pvepro.ru,newfi.ru -SOURCE_LABEL=pvepro.ru -ALERT_RECIPIENT_SHA256=0d4cf60ca8cd53a12ad50dc5419d3c9c32192f0852000eb10051fbdef52b04bf -CONTACT_RECIPIENT_HASH_SET_COUNT=1 -PREFLIGHT=PASS -ALERT_RECIPIENT_SHA256=0d4cf60ca8cd53a12ad50dc5419d3c9c32192f0852000eb10051fbdef52b04bf -GUEST_PREFLIGHT=PASS -GUEST_STDERR_PRESENT=false -=== VM210 PREFLIGHT === -VM210_STATE=running -VM210_QGA_TRANSPORT_RC=0 -GUEST_EXITCODE=0 -ROOTS=1 -DOMAINS=newfi.ru -FLEET_DOMAINS=codevipe.ru,dsmods.ru,gamevipe.ru,hkmods.ru,kingofwolk.ru,zakrutim.ru,pvepro.ru,newfi.ru -SOURCE_LABEL=newfi.ru -ALERT_RECIPIENT_SHA256=0d4cf60ca8cd53a12ad50dc5419d3c9c32192f0852000eb10051fbdef52b04bf -CONTACT_RECIPIENT_HASH_SET_COUNT=1 -PREFLIGHT=PASS -ALERT_RECIPIENT_SHA256=0d4cf60ca8cd53a12ad50dc5419d3c9c32192f0852000eb10051fbdef52b04bf -GUEST_PREFLIGHT=PASS -GUEST_STDERR_PRESENT=false -=== VM160 PREFLIGHT === -VM160_STATE=running -VM160_QGA_TRANSPORT_RC=0 -VM160_GUEST_EXITCODE=0 -VM160_MONITOR_PRE_SHA256=5d3055077aafb084c0f6424dcd33950f589296dc21a52024e7b79adc7e9c2dd1 -ROOTS=6 -DOMAINS=codevipe.ru,dsmods.ru,gamevipe.ru,hkmods.ru,kingofwolk.ru,zakrutim.ru -FLEET_DOMAINS=codevipe.ru,dsmods.ru,gamevipe.ru,hkmods.ru,kingofwolk.ru,zakrutim.ru,pvepro.ru,newfi.ru -SOURCE_LABEL=forum-prod -ALERT_RECIPIENT_SHA256=0d4cf60ca8cd53a12ad50dc5419d3c9c32192f0852000eb10051fbdef52b04bf -CONTACT_RECIPIENT_HASH_SET_COUNT=1 -PREFLIGHT=PASS -VM160_PREFLIGHT=PASS -VM160_GUEST_STDERR_PRESENT=false -CHANGES_MADE=true -HEALTH_POST_SHA256=5900ae2822bcb5abd7ec3c380f2e6d9a16658f0d7f5e3f8cc9ccbd62764b5337 -HEALTH_RUN_RC=0 -HEALTH_LINE=STATUS=OK TS=2026-09-23T22:08:25Z TYPE=forum-snuffleupagus BAD=0 WARN=0 MODE=low-noise-simulation STRICT=NO MODULE=loaded FORUMS=8 PUBLIC_SMOKE_BAD=0 ACTUAL_READONLY_EXEC_RULE=NO REPORT=/var/lib/homelab-forum-snuffleupagus/latest.md -=== VM160 APPLY === -VM160_STATE=running -VM160_QGA_TRANSPORT_RC=0 -VM160_GUEST_EXITCODE=0 -VM160_MONITOR_PRE_SHA256=5d3055077aafb084c0f6424dcd33950f589296dc21a52024e7b79adc7e9c2dd1 -ROOTS=6 -DOMAINS=codevipe.ru,dsmods.ru,gamevipe.ru,hkmods.ru,kingofwolk.ru,zakrutim.ru -FLEET_DOMAINS=codevipe.ru,dsmods.ru,gamevipe.ru,hkmods.ru,kingofwolk.ru,zakrutim.ru,pvepro.ru,newfi.ru -SOURCE_LABEL=forum-prod -ALERT_RECIPIENT_SHA256=0d4cf60ca8cd53a12ad50dc5419d3c9c32192f0852000eb10051fbdef52b04bf -CONTACT_RECIPIENT_HASH_SET_COUNT=1 -PREFLIGHT=PASS -VM160_PREFLIGHT=PASS -VM160_MONITOR_POST_SHA256=3417dc26c8292ac3b07be8aafb27bd5228d5876bda61bc1a0a38be4e85ea7052 -VM160_ALERT_TIMER_ACTIVE=active -VM160_ALERT_TIMER_ENABLED=enabled -VM160_APPLY=PASS -VM160_GUEST_STDERR_PRESENT=false -=== VM100 APPLY/CANARY === -VM100_STATE=running -VM100_QGA_TRANSPORT_RC=0 -GUEST_EXITCODE=90 -ROOTS=1 -DOMAINS=pvepro.ru -FLEET_DOMAINS=codevipe.ru,dsmods.ru,gamevipe.ru,hkmods.ru,kingofwolk.ru,zakrutim.ru,pvepro.ru,newfi.ru -SOURCE_LABEL=pvepro.ru -ALERT_RECIPIENT_SHA256=0d4cf60ca8cd53a12ad50dc5419d3c9c32192f0852000eb10051fbdef52b04bf -CONTACT_RECIPIENT_HASH_SET_COUNT=1 -PREFLIGHT=PASS -ALERT_RECIPIENT_SHA256=0d4cf60ca8cd53a12ad50dc5419d3c9c32192f0852000eb10051fbdef52b04bf -GUEST_PREFLIGHT=PASS -ROOTS=1 -DOMAINS=pvepro.ru -FLEET_DOMAINS=codevipe.ru,dsmods.ru,gamevipe.ru,hkmods.ru,kingofwolk.ru,zakrutim.ru,pvepro.ru,newfi.ru -SOURCE_LABEL=pvepro.ru -ALERT_RECIPIENT_SHA256=0d4cf60ca8cd53a12ad50dc5419d3c9c32192f0852000eb10051fbdef52b04bf -CONTACT_RECIPIENT_HASH_SET_COUNT=1 -INITIALIZE=PASS -ALERT_TIMER_ACTIVE=active -ALERT_TIMER_ENABLED=enabled -STATE_SITE_COUNT=1 -CANARY_BEGIN=true -ROOTS=1 -DOMAINS=pvepro.ru -FLEET_DOMAINS=codevipe.ru,dsmods.ru,gamevipe.ru,hkmods.ru,kingofwolk.ru,zakrutim.ru,pvepro.ru,newfi.ru -SOURCE_LABEL=pvepro.ru -ALERT_RECIPIENT_SHA256=0d4cf60ca8cd53a12ad50dc5419d3c9c32192f0852000eb10051fbdef52b04bf -CONTACT_RECIPIENT_HASH_SET_COUNT=1 -TEST_SEND_RC=65 -TEST_SEND_ERR=sendmail: envelope from address not accepted by the server sendmail: server message: 553 5.7.1 Sender address rejected: user not found. 1790201313-X8lFAQ7fpW20-VJHZAQ43 sendmail: could not send mail (account default from /etc/msmtprc) -ALERT_FAILURE=TEST_SEND_FAILED_65 -GUEST_FAILURE=CANARY_SEND -GUEST_ROLLBACK_STARTED=true -GUEST_ROLLBACK_RESTORED=true -GUEST_STDERR_PRESENT=true -GUEST_STDERR_SHA256=f704b93c5de99a394103c9ff88a07ffd86f97675d45caeb3a7829b72b3152eec -GUEST_STDERR_BYTES=154 -FAILURE=VM100_APPLY_RC_90 -ROLLBACK_STARTED=true -ROLLBACK_RESTORED=true -DECISION=FAIL_EIGHT_FORUM_EXPANSION_ROLLED_BACK -NEXT_GATE=REVIEW_FAILURE -TASK_RESULT=FAIL_EIGHT_FORUM_EXPANSION -HOMELAB_RESULT_CONTRACT={"version":1,"command_id":"FORUM-260924-EXPAND-8FORUM-ALERTS4A27R1","status":"FAIL","changes_made":true,"rollback_started":true,"rollback_restored":true} +MUTATION_OUTCOME=NO_MUTATION +SANITIZED=yes +SECRETS_INCLUDED=no +PRIVATE_ADDRESSES_INCLUDED=no +RAW_EVIDENCE_SHA256=06d7265e11af10711ebcbd148af88f0163f786ca78ac3be9eb2a43259e608e4c +SANITIZED_OUTPUT_SHA256=06d7265e11af10711ebcbd148af88f0163f786ca78ac3be9eb2a43259e608e4c +OUTPUT_BEGIN +WORKERS2_MOLDOVA_BACKUP_SSH_EXECUTOR_RCA_BEGIN=1 +COMMAND_ID=SUPPORT-260924-HOMELAB-MOLDOVA-BACKUP-SSH-EXECUTOR-RCA-1145R1 +MODE=read-only +COMPONENT=homelab-moldova-backup-ssh-executor-rca +MUTATION_BOUNDARY=NONE;PVE01_LOCAL_RELEASE_EXECUTOR_DISCOVERY_ONLY;NO_MOLDOVA_CONNECTION;NO_REMOTE_SSH;NO_PROVIDER_CALL;NO_SYSTEMD_ACTION;NO_BACKUP_RUN;NO_BACKUP_MUTATION;NO_TARGET_POLICY_MUTATION;NO_NETBIRD_MUTATION;NO_MONITORING_MUTATION;NO_GIT_WRITE;NO_SECRET_VALUE_READOUT +PREDECESSOR_1144=PASS_MOLDOVA_1144_SSH_AUTH_RCA_COLLECTED +PREDECESSOR_1144_AUTH_TRANSPORT_CLASS=OPENSSH_EFFECTIVE_IDENTITY_FILE_EXISTS +RCA_REASON=1144_PROVED_DEFAULT_IDENTITIES_EXIST_BUT_DID_NOT_DISCOVER_ACTUAL_RELEASE_EXECUTOR +CARRIER_BASELINE=KNOWN_GOOD_1137_BYTE_SEMANTICS +REFERENCE_SHA256=5a3d8e5154c41cb582a4e0aca68090be1f0138918bf82131a948df326f9d8d66 +ERROR_REGISTER_SHA256=3b09a553ec0f527ed3afeed4753f52a74ee3036045bbfb3c685e9f8af4ba7ba0 +RUNNER_SHA256=b248a4c32c9cc64e5747e7dce6c7fc0a23f5124a77c71ce72e27a81aceae9d2d +TARGETS_SHA256=5cba5f1e14c65211c9fab6d11594c19279ab68369b4db3f7283708c368348288 +MD_RELAY_COUNT=1 +MD_RELAY_ENABLED=false +MD_RELAY_TYPE=vps_md +MD_RELAY_SCOPE=vps +WRAPPER_META_BEGIN=1 +WRAPPER_PATH=/usr/local/sbin/homelab-backup-v2 +WRAPPER_SHA256=5077444065c732451cb84898680f62361b21a24ef9eb4f191253e82ead524ea2 +WRAPPER_BYTES=1656 +WRAPPER_MODE=750 +WRAPPER_OWNER=root:root +WRAPPER_META_END=1 +WRAPPER_DISCOVERY_SNIPPETS_BEGIN=1 +WRAPPER_SRC|line=1|text=#!/usr/bin/python3 +WRAPPER_SRC|line=4|text=ACCEPTED=pathlib.Path('/opt/homelab-backup-v2/releases/backup-v2-overlay-504r1-ca6876fa9c3b') +WRAPPER_SRC|line=5|text=EXPECTED={'fresh_lxc_helper.py': 'dbb3e6dbb71b10797df736431d9501a752a0f8c439163c9a0bddb7f76747f011', 'producer_exec_helper.py': 'b3b97099bcd7126670f866e81faae6a42a96d869d9989e6a67f6ea6c871c90b3', 'lxc_adapter.py': '3fa7d194010d76f7c8bc95035ae8876262e9337903e310f45f45df88c266dc8d', 'qemu_adapter.py': '1906db42bd34bbe85eb46fd9797b3572d46e0a41c29d1c29be5f9df5e448f15b', 'scheduler_adapter.py': 'a6f67159a64067773323f028126e3109e15cf17a8142cea5e6129970f1d4da96', 'entry504.py': 'ca6876fa9c3b1465b045344a8ad46962de16972c7dadc4ac6257c7d225047793'} +WRAPPER_SRC|line=7|text= h=hashlib.sha256() +WRAPPER_SRC|line=16|text= if not stat.S_ISDIR(st.st_mode) or stat.S_ISLNK(st.st_mode) or st.st_uid!=0 or st.st_gid!=0 or stat.S_IMODE(st.st_mode)!=0o500: raise SystemExit('RELEASE_ROOT_IDENTITY') +WRAPPER_SRC|line=18|text= if names!=sorted(EXPECTED): raise SystemExit('RELEASE_FILESET') +WRAPPER_SRC|line=21|text= if not stat.S_ISREG(s.st_mode) or stat.S_ISLNK(s.st_mode) or s.st_uid!=0 or s.st_gid!=0 or stat.S_IMODE(s.st_mode)!=0o400 or sha(p)!=want: raise SystemExit('RELEASE_FILE_IDENTITY_'+name) +WRAPPER_DISCOVERY_SNIPPETS_END=1 +RELEASE_LINKS_BEGIN=1 +RELEASE_PATH|path=/var/lib/homelab-backup-v2|type=directory|mode=700|owner=root:root +RELEASE_PATH|path=/etc/homelab-backup-v2|type=directory|mode=700|owner=root:root +RELEASE_LINKS_END=1 +EXECUTOR_MATCH_FILES_BEGIN=1 +EXECUTOR_MATCH_FILE_COUNT=2 +EXECUTOR_FILE|path=/etc/homelab-backup-v2/targets.json|mode=600|owner=root|group=root|bytes=7415|sha256=5cba5f1e14c65211c9fab6d11594c19279ab68369b4db3f7283708c368348288 +EXECUTOR_FILE|path=/var/lib/homelab-backup-v2/state/md-relay.json|mode=600|owner=root|group=root|bytes=615|sha256=24c6e2f71fb84f1f2dcd5e95cd68a527f2c811254f4be3e666985f1a36859f76 +EXECUTOR_MATCH_FILES_END=1 +EXECUTOR_AUTH_SNIPPETS_BEGIN=1 +EXECUTOR_SRC|path=/etc/homelab-backup-v2/targets.json|line=168|text= "hostkey_fingerprint": "SHA256:DXJGwunC5tEVvnC3nRxyayTN8FXmyJcVgf+Oa6udQsY", +EXECUTOR_SRC|path=/etc/homelab-backup-v2/targets.json|line=169|text= "logical_id": "md-relay", +EXECUTOR_SRC|path=/etc/homelab-backup-v2/targets.json|line=172|text= "type": "vps_md", +EXECUTOR_SRC|path=/etc/homelab-backup-v2/targets.json|line=180|text= "hostkey_fingerprint": "SHA256:Ef+HsZb8lIXoadvYBXuGKIL3H7NyJMmf9EYCoJRoSv8", +EXECUTOR_SRC|path=/etc/homelab-backup-v2/targets.json|line=192|text= "hostkey_fingerprint": "SHA256:805Q9eh/fePUci42dfY58I6q0lFiiPHn0/M+mckypmE", +EXECUTOR_SRC|path=/var/lib/homelab-backup-v2/state/md-relay.json|line=1|text={"consecutive_failures":0,"failure_backoff_seconds":0,"generation":"v2-md-relay-20260923T003832Z-8051ad720cce","green_verified":false,"last_attempt_epoch":1790123935,"last_success_epoch":1790123935,"logical_id":"md-relay","manifest_sha256":"b57ca8ee55d8622f6aeb044d3063fd0bb7cb6ada83f50511c87894d083a2a428","payload_sha256":"0d65b5ba587ec9f393e7a54af1df2e86b54332509da617d6bfe594b42d5fa3f8","replicas":[{"crypt":"pve03-mail-01-crypt","physical":"pve03-mail-01"},{"crypt":"pve01-mail-04-crypt","physical":"pve01-mail-04"}],"result":"SUCCESS","retry_after_epoch":0,"schema":1,"updated_at_utc":"2026-09-23T00:38:55Z"} +EXECUTOR_AUTH_SNIPPET_COUNT=6 +EXECUTOR_AUTH_SNIPPETS_END=1 +MD_RELAY_STATE_BEGIN=1 +MD_RELAY_STATE_SHA256=24c6e2f71fb84f1f2dcd5e95cd68a527f2c811254f4be3e666985f1a36859f76 +STATE_LOGICAL_ID=md-relay +STATE_RESULT=SUCCESS +STATE_GENERATION=v2-md-relay-20260923T003832Z-8051ad720cce +STATE_UPDATED_AT_UTC=2026-09-23T00:38:55Z +STATE_CONSECUTIVE_FAILURES=0 +STATE_FAILURE_BACKOFF_SECONDS=0 +STATE_REPLICA_COUNT=2 +MD_RELAY_STATE_END=1 +MD_RELAY_SUCCESS_WINDOW_BEGIN=1 +Sep 23 03:38:32 pve01 systemd[1]: Starting homelab-backup-v2.service - Homelab Backup V2 manifest-driven scheduler... +Sep 23 03:38:32 pve01 homelab-backup-v2[2925325]: SELFTEST_REPLACEMENT486=PASS +Sep 23 03:38:55 pve01 homelab-backup-v2[2925325]: SCHEDULE_TARGET=md-relay +Sep 23 03:38:55 pve01 homelab-backup-v2[2925325]: SCHEDULE_RESULT=SUCCESS +Sep 23 03:38:55 pve01 systemd[1]: homelab-backup-v2.service: Deactivated successfully. +Sep 23 03:38:55 pve01 systemd[1]: Finished homelab-backup-v2.service - Homelab Backup V2 manifest-driven scheduler. +Sep 23 03:38:55 pve01 systemd[1]: homelab-backup-v2.service: Consumed 3.449s CPU time, 120.2M memory peak. +MD_RELAY_SUCCESS_WINDOW_END=1 +AUTH_EVIDENCE_EXPLICIT_IDENTITY_FILE_COUNT=0 +AUTH_EVIDENCE_STRICT_HOSTKEY_FILE_COUNT=0 +AUTH_EVIDENCE_USER_KNOWN_HOSTS_FILE_COUNT=0 +AUTH_EVIDENCE_BATCHMODE_FILE_COUNT=0 +AUTH_EVIDENCE_OPS_USER_FILE_COUNT=1 +BACKUPV2_MD_RELAY_AUTH_CLASS=EXECUTOR_FOUND_AUTH_OPTIONS_NOT_FULLY_CLASSIFIED +DECISION=PASS_MOLDOVA_1145_BACKUP_SSH_EXECUTOR_RCA +NEXT_GATE=READ_EXACT_1145_AND_BUILD_REMOTE_ROLE_PREFLIGHT_WITH_EXACT_EXECUTOR_AUTH +TASK_RESULT=PASS_MOLDOVA_BACKUP_SSH_EXECUTOR_RCA +HOMELAB_RESULT_CONTRACT={"version":1,"command_id":"SUPPORT-260924-HOMELAB-MOLDOVA-BACKUP-SSH-EXECUTOR-RCA-1145R1","status":"OK","changes_made":false,"rollback_started":false,"rollback_restored":null} +WORKERS2_MOLDOVA_BACKUP_SSH_EXECUTOR_RCA_END=1 OUTPUT_END CHAT_OUTPUT_END