runtime: publish SUPPORT-260928-HOMELAB-BACKUP-AUDIT-NOISE-FIX-1239R1
This commit is contained in:
@@ -0,0 +1,38 @@
|
||||
{
|
||||
"schema_version": 1,
|
||||
"channel": "homelab-runtime",
|
||||
"command_id": "SUPPORT-260928-HOMELAB-BACKUP-AUDIT-NOISE-FIX-1239R1",
|
||||
"status": "OK",
|
||||
"rc": 0,
|
||||
"host": "pve01",
|
||||
"mode": "change",
|
||||
"component": "homelab-backup-audit-noise-fix",
|
||||
"started_at_utc": "2026-09-28T04:47:36Z",
|
||||
"finished_at_utc": "2026-09-28T04:47:37Z",
|
||||
"reference_register_checked": true,
|
||||
"reference_sha256": "5a3d8e5154c41cb582a4e0aca68090be1f0138918bf82131a948df326f9d8d66",
|
||||
"error_register_checked": true,
|
||||
"error_register_sha256": "3b09a553ec0f527ed3afeed4753f52a74ee3036045bbfb3c685e9f8af4ba7ba0",
|
||||
"command_sha256": "eaf5d26b4c7d4527c127f18a975124a32d11d0a248303f6d508213a86112b6d7",
|
||||
"duplicate_failed_command_blocked": false,
|
||||
"block_reason": null,
|
||||
"execution_started": true,
|
||||
"change_declared": true,
|
||||
"result_contract_valid": true,
|
||||
"result_contract_status": "OK",
|
||||
"result_contract_error": null,
|
||||
"command_rc": 0,
|
||||
"changes_made": false,
|
||||
"rollback_started": false,
|
||||
"rollback_restored": null,
|
||||
"mutation_outcome": "NO_MUTATION",
|
||||
"sanitized": true,
|
||||
"secrets_included": false,
|
||||
"private_addresses_included": false,
|
||||
"raw_evidence_retained_locally": true,
|
||||
"raw_evidence_sha256": "48449c1356cf37930a97abb62f957eb59e29701542f9ef44081dde1eea9395e8",
|
||||
"sanitized_output_sha256": "48449c1356cf37930a97abb62f957eb59e29701542f9ef44081dde1eea9395e8",
|
||||
"output_truncated_in_json": false,
|
||||
"full_sanitized_output_url": "https://git.gram1.ru/.well-known/homelab-runtime/latest.txt",
|
||||
"output": "BACKUP_AUDIT_NOISE_FIX_BEGIN=true\nCOMMAND_ID=SUPPORT-260928-HOMELAB-BACKUP-AUDIT-NOISE-FIX-1239R1\nMODE=change\nCOMPONENT=homelab-backup-audit-noise-fix\nMUTATION_BOUNDARY=ONLY_BACKUP_AUDIT_FALSE_NOISE_LOGIC_PLUS_CANONICAL_SOURCE_COMMIT_PUSH_AND_CONTROLLED_HEALTH_REFRESH;NO_BACKUP_ENGINE_CHANGE;NO_ALERT_MAILER_CHANGE;NO_TIMER_CHANGE;NO_SYSTEMD_ACTION;NO_BACKUP;NO_RETENTION;NO_DELETE;NO_RESTORE;NO_MANUAL_MAIL;NO_NETBIRD_SKLADCHIK_CHANGE\n=== AUTHORITY ===\nHOST=pve01\nCLUSTER_QUORATE=Yes\nRUNNER_SHA256=b248a4c32c9cc64e5747e7dce6c7fc0a23f5124a77c71ce72e27a81aceae9d2d\nSOURCE_1238_STATUS=OK\nSOURCE_1238_RC=0\nSOURCE_1238_DECISION=PASS_BACKUP_ALERT_NOISE_ROOT_RCA_COLLECTED\nSOURCE_1238_NEXT_GATE=READ_EXACT_1238_AND_APPLY_ONLY_PROVEN_ALERT_NOISE_FIX\n=== LIVE_PRECONDITIONS ===\nWRAPPER_SHA256=894f8ea346b53a3d7846dbd6eeb5b89333957eb6d7a4f2f64fd4557cae176e1c\nHELPER_SHA256=4e39404b1ddb97392645adce6280f697182cca03714d3a462d363eebce9b1860\nVZDUMP_CONF_SHA256=027e4b04852a4b83d558eb861d432b6fad4f234e12ab77d327f274d0fdedbfad\nAUDIT_SHA256=f7b936fa997bdefdd0c3c5d1d96e496888ebebc8ea85a60ed3bf5bcc9bf4962a\nALERT_SHA256=2b92cd44fc5b43decba0ea0706c4c84e1912cdc6d7b9bcb10bbf9552c98dc3f9\nSOURCE_PRE_HEAD=db592a25c6ac587bf16472a6132b66698c42f663\nSOURCE_BRANCH=cr-2026-1005-backup-v2-health-alerting-remediation\nSOURCE_REMOTE_HEAD_PRE=db592a25c6ac587bf16472a6132b66698c42f663\nSOURCE_WORKTREE_CLEAN=true\nCANON_AUDIT_SHA256=f7b936fa997bdefdd0c3c5d1d96e496888ebebc8ea85a60ed3bf5bcc9bf4962a\nCANON_ALERT_SHA256=2b92cd44fc5b43decba0ea0706c4c84e1912cdc6d7b9bcb10bbf9552c98dc3f9\n=== BUILD_PATCHED_AUDIT ===\nPATCH_BUILD_RC=0\nNEW_AUDIT_SHA256=1c2cd8a00859492c010ed35baf5eb67f4278544809de8c2563421a6f3e356ed3\nSTATIC_EXPECTED_OWNER_CALL_COUNT=2\nSTATIC_GIT_HOME_SET_COUNT=1\nTASK_RESULT=HOLD_BACKUP_AUDIT_NOISE_FIX\nDECISION=HOLD_1239_STATIC_PATCH_VERIFY_FAILED\nNEXT_GATE=FIX_ONLY_PATCH_BUILDER\nHOMELAB_RESULT_CONTRACT={\"version\":1,\"command_id\":\"SUPPORT-260928-HOMELAB-BACKUP-AUDIT-NOISE-FIX-1239R1\",\"status\":\"OK\",\"changes_made\":false,\"rollback_started\":false,\"rollback_restored\":null}\n"
|
||||
}
|
||||
@@ -0,0 +1,66 @@
|
||||
CHAT_OUTPUT_BEGIN
|
||||
COMMAND_ID=SUPPORT-260928-HOMELAB-BACKUP-AUDIT-NOISE-FIX-1239R1
|
||||
STATUS=OK
|
||||
RC=0
|
||||
HOST=pve01
|
||||
MODE=change
|
||||
COMPONENT=homelab-backup-audit-noise-fix
|
||||
REFERENCE_REGISTER_CHECK=OK
|
||||
REFERENCE_SHA256=5a3d8e5154c41cb582a4e0aca68090be1f0138918bf82131a948df326f9d8d66
|
||||
ERROR_REGISTER_CHECK=OK
|
||||
ERROR_REGISTER_SHA256=3b09a553ec0f527ed3afeed4753f52a74ee3036045bbfb3c685e9f8af4ba7ba0
|
||||
COMMAND_SHA256=eaf5d26b4c7d4527c127f18a975124a32d11d0a248303f6d508213a86112b6d7
|
||||
DUPLICATE_FAILED_COMMAND_BLOCKED=false
|
||||
EXECUTION_STARTED=true
|
||||
CHANGE_DECLARED=true
|
||||
RESULT_CONTRACT_VALID=true
|
||||
RESULT_CONTRACT_STATUS=OK
|
||||
RESULT_CONTRACT_ERROR=NONE
|
||||
COMMAND_RC=0
|
||||
CHANGES_MADE=false
|
||||
ROLLBACK_STARTED=false
|
||||
ROLLBACK_RESTORED=null
|
||||
MUTATION_OUTCOME=NO_MUTATION
|
||||
SANITIZED=yes
|
||||
SECRETS_INCLUDED=no
|
||||
PRIVATE_ADDRESSES_INCLUDED=no
|
||||
RAW_EVIDENCE_SHA256=48449c1356cf37930a97abb62f957eb59e29701542f9ef44081dde1eea9395e8
|
||||
SANITIZED_OUTPUT_SHA256=48449c1356cf37930a97abb62f957eb59e29701542f9ef44081dde1eea9395e8
|
||||
OUTPUT_BEGIN
|
||||
BACKUP_AUDIT_NOISE_FIX_BEGIN=true
|
||||
COMMAND_ID=SUPPORT-260928-HOMELAB-BACKUP-AUDIT-NOISE-FIX-1239R1
|
||||
MODE=change
|
||||
COMPONENT=homelab-backup-audit-noise-fix
|
||||
MUTATION_BOUNDARY=ONLY_BACKUP_AUDIT_FALSE_NOISE_LOGIC_PLUS_CANONICAL_SOURCE_COMMIT_PUSH_AND_CONTROLLED_HEALTH_REFRESH;NO_BACKUP_ENGINE_CHANGE;NO_ALERT_MAILER_CHANGE;NO_TIMER_CHANGE;NO_SYSTEMD_ACTION;NO_BACKUP;NO_RETENTION;NO_DELETE;NO_RESTORE;NO_MANUAL_MAIL;NO_NETBIRD_SKLADCHIK_CHANGE
|
||||
=== AUTHORITY ===
|
||||
HOST=pve01
|
||||
CLUSTER_QUORATE=Yes
|
||||
RUNNER_SHA256=b248a4c32c9cc64e5747e7dce6c7fc0a23f5124a77c71ce72e27a81aceae9d2d
|
||||
SOURCE_1238_STATUS=OK
|
||||
SOURCE_1238_RC=0
|
||||
SOURCE_1238_DECISION=PASS_BACKUP_ALERT_NOISE_ROOT_RCA_COLLECTED
|
||||
SOURCE_1238_NEXT_GATE=READ_EXACT_1238_AND_APPLY_ONLY_PROVEN_ALERT_NOISE_FIX
|
||||
=== LIVE_PRECONDITIONS ===
|
||||
WRAPPER_SHA256=894f8ea346b53a3d7846dbd6eeb5b89333957eb6d7a4f2f64fd4557cae176e1c
|
||||
HELPER_SHA256=4e39404b1ddb97392645adce6280f697182cca03714d3a462d363eebce9b1860
|
||||
VZDUMP_CONF_SHA256=027e4b04852a4b83d558eb861d432b6fad4f234e12ab77d327f274d0fdedbfad
|
||||
AUDIT_SHA256=f7b936fa997bdefdd0c3c5d1d96e496888ebebc8ea85a60ed3bf5bcc9bf4962a
|
||||
ALERT_SHA256=2b92cd44fc5b43decba0ea0706c4c84e1912cdc6d7b9bcb10bbf9552c98dc3f9
|
||||
SOURCE_PRE_HEAD=db592a25c6ac587bf16472a6132b66698c42f663
|
||||
SOURCE_BRANCH=cr-2026-1005-backup-v2-health-alerting-remediation
|
||||
SOURCE_REMOTE_HEAD_PRE=db592a25c6ac587bf16472a6132b66698c42f663
|
||||
SOURCE_WORKTREE_CLEAN=true
|
||||
CANON_AUDIT_SHA256=f7b936fa997bdefdd0c3c5d1d96e496888ebebc8ea85a60ed3bf5bcc9bf4962a
|
||||
CANON_ALERT_SHA256=2b92cd44fc5b43decba0ea0706c4c84e1912cdc6d7b9bcb10bbf9552c98dc3f9
|
||||
=== BUILD_PATCHED_AUDIT ===
|
||||
PATCH_BUILD_RC=0
|
||||
NEW_AUDIT_SHA256=1c2cd8a00859492c010ed35baf5eb67f4278544809de8c2563421a6f3e356ed3
|
||||
STATIC_EXPECTED_OWNER_CALL_COUNT=2
|
||||
STATIC_GIT_HOME_SET_COUNT=1
|
||||
TASK_RESULT=HOLD_BACKUP_AUDIT_NOISE_FIX
|
||||
DECISION=HOLD_1239_STATIC_PATCH_VERIFY_FAILED
|
||||
NEXT_GATE=FIX_ONLY_PATCH_BUILDER
|
||||
HOMELAB_RESULT_CONTRACT={"version":1,"command_id":"SUPPORT-260928-HOMELAB-BACKUP-AUDIT-NOISE-FIX-1239R1","status":"OK","changes_made":false,"rollback_started":false,"rollback_restored":null}
|
||||
|
||||
OUTPUT_END
|
||||
CHAT_OUTPUT_END
|
||||
+11
-11
File diff suppressed because one or more lines are too long
+31
-203
@@ -1,20 +1,20 @@
|
||||
CHAT_OUTPUT_BEGIN
|
||||
COMMAND_ID=SUPPORT-260928-HOMELAB-BACKUP-ALERT-NOISE-ROOT-RCA-1238R1
|
||||
COMMAND_ID=SUPPORT-260928-HOMELAB-BACKUP-AUDIT-NOISE-FIX-1239R1
|
||||
STATUS=OK
|
||||
RC=0
|
||||
HOST=pve01
|
||||
MODE=read-only
|
||||
COMPONENT=homelab-backup-alert-noise-root-rca
|
||||
MODE=change
|
||||
COMPONENT=homelab-backup-audit-noise-fix
|
||||
REFERENCE_REGISTER_CHECK=OK
|
||||
REFERENCE_SHA256=5a3d8e5154c41cb582a4e0aca68090be1f0138918bf82131a948df326f9d8d66
|
||||
ERROR_REGISTER_CHECK=OK
|
||||
ERROR_REGISTER_SHA256=3b09a553ec0f527ed3afeed4753f52a74ee3036045bbfb3c685e9f8af4ba7ba0
|
||||
COMMAND_SHA256=9ae99a8877a8c63075030326f98f8146f9c933c9cbcb7bd076b9bbcfeeb0e1a3
|
||||
COMMAND_SHA256=eaf5d26b4c7d4527c127f18a975124a32d11d0a248303f6d508213a86112b6d7
|
||||
DUPLICATE_FAILED_COMMAND_BLOCKED=false
|
||||
EXECUTION_STARTED=true
|
||||
CHANGE_DECLARED=false
|
||||
CHANGE_DECLARED=true
|
||||
RESULT_CONTRACT_VALID=true
|
||||
RESULT_CONTRACT_STATUS=NOT_APPLICABLE
|
||||
RESULT_CONTRACT_STATUS=OK
|
||||
RESULT_CONTRACT_ERROR=NONE
|
||||
COMMAND_RC=0
|
||||
CHANGES_MADE=false
|
||||
@@ -24,215 +24,43 @@ MUTATION_OUTCOME=NO_MUTATION
|
||||
SANITIZED=yes
|
||||
SECRETS_INCLUDED=no
|
||||
PRIVATE_ADDRESSES_INCLUDED=no
|
||||
RAW_EVIDENCE_SHA256=9c2e7b15aaf9e4270699137f0f21aa91d7e88a1ae44c347b734ba55e3bbe2b81
|
||||
SANITIZED_OUTPUT_SHA256=9c2e7b15aaf9e4270699137f0f21aa91d7e88a1ae44c347b734ba55e3bbe2b81
|
||||
RAW_EVIDENCE_SHA256=48449c1356cf37930a97abb62f957eb59e29701542f9ef44081dde1eea9395e8
|
||||
SANITIZED_OUTPUT_SHA256=48449c1356cf37930a97abb62f957eb59e29701542f9ef44081dde1eea9395e8
|
||||
OUTPUT_BEGIN
|
||||
BACKUP_ALERT_NOISE_ROOT_RCA_BEGIN=true
|
||||
COMMAND_ID=SUPPORT-260928-HOMELAB-BACKUP-ALERT-NOISE-ROOT-RCA-1238R1
|
||||
MODE=read-only
|
||||
COMPONENT=homelab-backup-alert-noise-root-rca
|
||||
MUTATION_BOUNDARY=READ_ONLY_SYSTEMD_CONTEXT_GIT_REMOTE_PROBES_AUDIT_ALL_SOURCE_DISCOVERY_AND_HISTORICAL_CORRELATION;NO_ALERT_EXECUTION;NO_MAIL;NO_BACKUP;NO_LOCK_ACQUIRE;NO_CONFIG_WRITE;NO_SCRIPT_WRITE;NO_SYSTEMD_ACTION;NO_TIMER_CHANGE;NO_HEALTH_WRITE;NO_GIT_WRITE;NO_RETENTION;NO_DELETE;NO_RESTORE;NO_NETBIRD_SKLADCHIK_CHANGE
|
||||
BACKUP_AUDIT_NOISE_FIX_BEGIN=true
|
||||
COMMAND_ID=SUPPORT-260928-HOMELAB-BACKUP-AUDIT-NOISE-FIX-1239R1
|
||||
MODE=change
|
||||
COMPONENT=homelab-backup-audit-noise-fix
|
||||
MUTATION_BOUNDARY=ONLY_BACKUP_AUDIT_FALSE_NOISE_LOGIC_PLUS_CANONICAL_SOURCE_COMMIT_PUSH_AND_CONTROLLED_HEALTH_REFRESH;NO_BACKUP_ENGINE_CHANGE;NO_ALERT_MAILER_CHANGE;NO_TIMER_CHANGE;NO_SYSTEMD_ACTION;NO_BACKUP;NO_RETENTION;NO_DELETE;NO_RESTORE;NO_MANUAL_MAIL;NO_NETBIRD_SKLADCHIK_CHANGE
|
||||
=== AUTHORITY ===
|
||||
HOST=pve01
|
||||
CLUSTER_QUORATE=Yes
|
||||
RUNNER_SHA256=b248a4c32c9cc64e5747e7dce6c7fc0a23f5124a77c71ce72e27a81aceae9d2d
|
||||
SOURCE_1237_STATUS=OK
|
||||
SOURCE_1237_RC=0
|
||||
SOURCE_1237_DECISION=PASS_BACKUP_ALERT_SPAM_CURRENT_RCA_COLLECTED
|
||||
SOURCE_1237_NEXT_GATE=READ_EXACT_1237_AND_FIX_ONLY_PROVEN_ALERT_NOISE_CAUSE
|
||||
=== LIVE_IDENTITIES ===
|
||||
SOURCE_1238_STATUS=OK
|
||||
SOURCE_1238_RC=0
|
||||
SOURCE_1238_DECISION=PASS_BACKUP_ALERT_NOISE_ROOT_RCA_COLLECTED
|
||||
SOURCE_1238_NEXT_GATE=READ_EXACT_1238_AND_APPLY_ONLY_PROVEN_ALERT_NOISE_FIX
|
||||
=== LIVE_PRECONDITIONS ===
|
||||
WRAPPER_SHA256=894f8ea346b53a3d7846dbd6eeb5b89333957eb6d7a4f2f64fd4557cae176e1c
|
||||
HELPER_SHA256=4e39404b1ddb97392645adce6280f697182cca03714d3a462d363eebce9b1860
|
||||
VZDUMP_CONF_SHA256=027e4b04852a4b83d558eb861d432b6fad4f234e12ab77d327f274d0fdedbfad
|
||||
AUDIT_SHA256=f7b936fa997bdefdd0c3c5d1d96e496888ebebc8ea85a60ed3bf5bcc9bf4962a
|
||||
ALERT_SHA256=2b92cd44fc5b43decba0ea0706c4c84e1912cdc6d7b9bcb10bbf9552c98dc3f9
|
||||
=== SOURCE_BRANCH_AUTHORITY ===
|
||||
SOURCE_LOCAL_HEAD=db592a25c6ac587bf16472a6132b66698c42f663
|
||||
SOURCE_PRE_HEAD=db592a25c6ac587bf16472a6132b66698c42f663
|
||||
SOURCE_BRANCH=cr-2026-1005-backup-v2-health-alerting-remediation
|
||||
SOURCE_REMOTE_HEAD_PRE=db592a25c6ac587bf16472a6132b66698c42f663
|
||||
SOURCE_WORKTREE_CLEAN=true
|
||||
SOURCE_ORIGIN_SCHEME=https
|
||||
SOURCE_ORIGIN_URL_SHA256=7507b97d441d05c4e33306608cbf655e96bf2add0ec7683cc2f4e5539f64daf9
|
||||
SOURCE_REMOTE_HEAD=db592a25c6ac587bf16472a6132b66698c42f663
|
||||
SOURCE_LOCAL_REMOTE_EXACT=true
|
||||
=== SYSTEMD_UNIT_CONTEXT ===
|
||||
SYSTEMD_SHOW_BEGIN=homelab-backup-audit.service
|
||||
ExecStart={ path=/usr/local/sbin/homelab-backup-audit ; argv[]=/usr/local/sbin/homelab-backup-audit scheduled ; ignore_errors=no ; start_time=[Mon 2026-09-28 07:36:16 MSK] ; stop_time=[Mon 2026-09-28 07:36:20 MSK] ; pid=2104808 ; code=exited ; status=1 }
|
||||
Environment=
|
||||
WorkingDirectory=
|
||||
RootDirectory=
|
||||
User=root
|
||||
Group=root
|
||||
DynamicUser=no
|
||||
PAMName=
|
||||
SYSTEMD_SHOW_END=homelab-backup-audit.service
|
||||
SYSTEMD_CAT_BEGIN=homelab-backup-audit.service
|
||||
# /etc/systemd/system/homelab-backup-audit.service
|
||||
[Unit]
|
||||
Description=Homelab unified backup and DR audit
|
||||
After=network-online.target
|
||||
Wants=network-online.target
|
||||
|
||||
[Service]
|
||||
Type=oneshot
|
||||
ExecStart=/usr/local/sbin/homelab-backup-audit scheduled
|
||||
User=root
|
||||
Group=root
|
||||
UMask=0077
|
||||
Nice=10
|
||||
IOSchedulingClass=best-effort
|
||||
IOSchedulingPriority=7
|
||||
SYSTEMD_CAT_END=homelab-backup-audit.service
|
||||
SYSTEMD_SHOW_BEGIN=homelab-alertmanager-backup-health.service
|
||||
ExecStart={ path=/usr/local/sbin/homelab-alertmanager-backup-run ; argv[]=/usr/local/sbin/homelab-alertmanager-backup-run health-alert ; ignore_errors=no ; start_time=[Mon 2026-09-28 07:31:56 MSK] ; stop_time=[Mon 2026-09-28 07:32:00 MSK] ; pid=2089737 ; code=exited ; status=0 }
|
||||
Environment=
|
||||
WorkingDirectory=
|
||||
RootDirectory=
|
||||
User=
|
||||
Group=
|
||||
DynamicUser=no
|
||||
PAMName=
|
||||
SYSTEMD_SHOW_END=homelab-alertmanager-backup-health.service
|
||||
SYSTEMD_CAT_BEGIN=homelab-alertmanager-backup-health.service
|
||||
# /etc/systemd/system/homelab-alertmanager-backup-health.service
|
||||
[Unit]
|
||||
Description=Workers2 backup health alert via existing alertmanager backup integration
|
||||
After=network-online.target
|
||||
|
||||
[Service]
|
||||
Type=oneshot
|
||||
ExecStart=/usr/local/sbin/homelab-alertmanager-backup-run health-alert
|
||||
SYSTEMD_CAT_END=homelab-alertmanager-backup-health.service
|
||||
SYSTEM_MANAGER_ENV_RELEVANT_BEGIN
|
||||
PATH=/usr/local/sbin:/usr/local/bin:/usr/sbin:/usr/bin
|
||||
SYSTEM_MANAGER_ENV_RELEVANT_END
|
||||
=== GIT_CONFIG_CONTEXT ===
|
||||
GIT_CONFIG_MODE=minimal OUT_BYTES=0 ERR_BYTES=21 OUT_SHA256=e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855 ERR_SHA256=6a8a72c20480deb7191693468ae1348ea123ecd0c5ff9baf8693dd293e9112cb
|
||||
GIT_CONFIG_MODE=nohome OUT_BYTES=0 ERR_BYTES=21 OUT_SHA256=e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855 ERR_SHA256=6a8a72c20480deb7191693468ae1348ea123ecd0c5ff9baf8693dd293e9112cb
|
||||
GIT_CONFIG_MODE=homeroot OUT_BYTES=346 ERR_BYTES=0 OUT_SHA256=9a188d2d29e9701bac3d2fe0cc4f11283f36fca0a941b1b3aeaf10d3b47fdc7e ERR_SHA256=e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855
|
||||
=== REMOTE_CONTEXT_PROBES ===
|
||||
REMOTE_PROBE=normal RC=0 HEAD=db592a25c6ac587bf16472a6132b66698c42f663 EXACT=true STDERR_BYTES=0 STDERR_SHA256=e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855
|
||||
REMOTE_PROBE=minimal RC=128 HEAD=NONE EXACT=false STDERR_BYTES=85 STDERR_SHA256=adbd757d7820290ef0868baee0cb62cc4953c3831e45981d82cb0ed5faed79cc
|
||||
REMOTE_PROBE_STDERR_BEGIN=minimal
|
||||
fatal: could not read Username for 'https://git.gram1.ru': No such device or address
|
||||
REMOTE_PROBE_STDERR_END=minimal
|
||||
REMOTE_PROBE=nohome RC=128 HEAD=NONE EXACT=false STDERR_BYTES=85 STDERR_SHA256=adbd757d7820290ef0868baee0cb62cc4953c3831e45981d82cb0ed5faed79cc
|
||||
REMOTE_PROBE_STDERR_BEGIN=nohome
|
||||
fatal: could not read Username for 'https://git.gram1.ru': No such device or address
|
||||
REMOTE_PROBE_STDERR_END=nohome
|
||||
REMOTE_PROBE=homeroot RC=0 HEAD=db592a25c6ac587bf16472a6132b66698c42f663 EXACT=true STDERR_BYTES=0 STDERR_SHA256=e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855
|
||||
=== AUDIT_ALL_CONTEXT_PROBES ===
|
||||
AUDIT_CONTEXT=minimal RC=1 SOURCE_OFFHOST_ISSUES=1 LOCK_BUSY_ISSUES=1 STDERR_BYTES=0
|
||||
AUDIT_CONTEXT_SUMMARY=minimal|SUMMARY|AUDIT_STATUS=OK|BACKUP_STATUS=RED|ACTIVE_COUNT=16|ISSUE_COUNT=2|CLOUD_A=OK|CLOUD_B=OK|CLOUD_B_DRILL=OK|DR_PROVEN=YES|SOURCE_OFFHOST=PENDING
|
||||
ISSUE|severity=MEDIUM|code=SOURCE_OFFHOST_MISSING|logical_id=-|root_cause=CR_BRANCH_SHA_NOT_EXACT
|
||||
AUDIT_CONTEXT=nohome RC=1 SOURCE_OFFHOST_ISSUES=1 LOCK_BUSY_ISSUES=1 STDERR_BYTES=0
|
||||
AUDIT_CONTEXT_SUMMARY=nohome|SUMMARY|AUDIT_STATUS=OK|BACKUP_STATUS=RED|ACTIVE_COUNT=16|ISSUE_COUNT=2|CLOUD_A=OK|CLOUD_B=OK|CLOUD_B_DRILL=OK|DR_PROVEN=YES|SOURCE_OFFHOST=PENDING
|
||||
ISSUE|severity=MEDIUM|code=SOURCE_OFFHOST_MISSING|logical_id=-|root_cause=CR_BRANCH_SHA_NOT_EXACT
|
||||
AUDIT_CONTEXT=homeroot RC=1 SOURCE_OFFHOST_ISSUES=0 LOCK_BUSY_ISSUES=1 STDERR_BYTES=0
|
||||
AUDIT_CONTEXT_SUMMARY=homeroot|SUMMARY|AUDIT_STATUS=OK|BACKUP_STATUS=RED|ACTIVE_COUNT=16|ISSUE_COUNT=1|CLOUD_A=OK|CLOUD_B=OK|CLOUD_B_DRILL=OK|DR_PROVEN=YES|SOURCE_OFFHOST=OK
|
||||
AUDIT_CONTEXT=normal RC=1 SOURCE_OFFHOST_ISSUES=0 LOCK_BUSY_ISSUES=1 STDERR_BYTES=0
|
||||
AUDIT_CONTEXT_SUMMARY=normal|SUMMARY|AUDIT_STATUS=OK|BACKUP_STATUS=RED|ACTIVE_COUNT=16|ISSUE_COUNT=1|CLOUD_A=OK|CLOUD_B=OK|CLOUD_B_DRILL=OK|DR_PROVEN=YES|SOURCE_OFFHOST=OK
|
||||
AUDIT_ALL_CONTEXT_HEALTH_NO_WRITE=true
|
||||
=== LOCK_OWNER_CORRELATION ===
|
||||
CURRENT_BACKUP_ACTIVE=activating
|
||||
CURRENT_BACKUP_MAINPID=1856481
|
||||
CURRENT_BACKUP_RESULT=success
|
||||
CURRENT_BACKUP_EXEC_STATUS=0
|
||||
homelab-backup- 1856481 FLOCK WRITE /run/homelab-backup-heavy.lock
|
||||
CURRENT_HEAVY_FUSER= 1856481
|
||||
RAW1237_PATH=/var/lib/homelab-chat-run/raw/SUPPORT-260928-HOMELAB-BACKUP-ALERT-SPAM-CURRENT-RCA-1237R1-20260928T043051Z.log
|
||||
25:UNIT=homelab-backup-v2.service ACTIVE=activating ENABLED=static RESULT=success EXEC_MAIN_STATUS=0 MAINPID=1856481
|
||||
238:LOCK_FUSER=/run/homelab-backup-heavy.lock| 1856481
|
||||
239:LOCK_FREE=/run/homelab-backup-heavy.lock|false
|
||||
249:homelab-backup- 1856481 FLOCK WRITE /run/homelab-backup-heavy.lock
|
||||
=== AUDIT_LOCK_LOGIC_STATIC ===
|
||||
LOCK_LOOP_SOURCE_BEGIN
|
||||
for lock in ["/run/homelab-backup-heavy.lock","/run/lock/homelab-app-cloud-quorum-queue.lock","/run/lock/homelab-vm-cloud-quorum-queue.lock","/run/lock/homelab-backup-dispatcher.lock"]:
|
||||
p=pathlib.Path(lock)
|
||||
if p.exists():
|
||||
lrc,_,_=run(["flock","-n",str(p),"-c","true"],5)
|
||||
if lrc!=0: issues.append({"severity":"HIGH","code":"LOCK_BUSY","logical_id":lock,"root_cause":"ACTIVE_OR_ORPHAN_LOCK"})
|
||||
LOCK_LOOP_SOURCE_END
|
||||
LOCK_LOOP_CALLS=issues.append,p.exists,pathlib.Path,run,str
|
||||
=== ALERT_COUNTER_LOGIC_STATIC ===
|
||||
ALERT_LOGIC|0028|if mode!="health-alert": raise SystemExit(64)
|
||||
ALERT_LOGIC|0029|p=subprocess.run([AUDIT,"write-health"],text=True,stdout=subprocess.PIPE,stderr=subprocess.PIPE)
|
||||
ALERT_LOGIC|0030|if p.returncode not in (0,1): raise SystemExit(p.returncode or 75)
|
||||
ALERT_LOGIC|0031|try:
|
||||
ALERT_LOGIC|0032| h=json.loads(HEALTH.read_text(encoding="utf-8"))
|
||||
ALERT_LOGIC|0033|except Exception:
|
||||
ALERT_LOGIC|0034| raise SystemExit(69)
|
||||
ALERT_LOGIC|0035|issues=h.get("issues") if isinstance(h.get("issues"),list) else []
|
||||
ALERT_LOGIC|0036|cur={key(i):i for i in issues if isinstance(i,dict)}
|
||||
ALERT_LOGIC|0037|codes=set(str(i.get("code","")) for i in cur.values())
|
||||
ALERT_LOGIC|0038|oldj={}
|
||||
ALERT_LOGIC|0039|if STATE.is_file():
|
||||
ALERT_LOGIC|0040| try:
|
||||
ALERT_LOGIC|0041| oldj=json.loads(STATE.read_text(encoding="utf-8"))
|
||||
ALERT_LOGIC|0042| if not isinstance(oldj,dict): oldj={}
|
||||
ALERT_LOGIC|0043| except Exception:
|
||||
ALERT_LOGIC|0044| oldj={}
|
||||
ALERT_LOGIC|0045|migrating=oldj.get("schema")!=SCHEMA
|
||||
ALERT_LOGIC|0046|prev_counts=oldj.get("code_counts") if isinstance(oldj.get("code_counts"),dict) and not migrating else {}
|
||||
ALERT_LOGIC|0047|counts={}
|
||||
ALERT_LOGIC|0048|for code in codes:
|
||||
ALERT_LOGIC|0049| try: prev=int(prev_counts.get(code,0))
|
||||
ALERT_LOGIC|0050| except Exception: prev=0
|
||||
ALERT_LOGIC|0051| counts[code]=prev+1
|
||||
ALERT_LOGIC|0052|actionable={}
|
||||
ALERT_LOGIC|0053|for k,i in cur.items():
|
||||
ALERT_LOGIC|0054| code=str(i.get("code",""))
|
||||
ALERT_LOGIC|0055| need=int(THRESHOLDS.get(code,1))
|
||||
ALERT_LOGIC|0056| if counts.get(code,0)>=need:
|
||||
ALERT_LOGIC|0057| actionable[k]=i
|
||||
ALERT_LOGIC|0058|tokens=[REDACTED]
|
||||
ALERT_LOGIC|0082|STATE.parent.mkdir(parents=True,exist_ok=True)
|
||||
ALERT_LOGIC|0083|tmp=STATE.with_name(STATE.name+".tmp")
|
||||
ALERT_LOGIC|0084|payload={
|
||||
ALERT_LOGIC|0085| "schema":SCHEMA,
|
||||
ALERT_LOGIC|0086| "updated_epoch":int(time.time()),
|
||||
ALERT_LOGIC|0087| "issue_keys":sorted(actionable),
|
||||
ALERT_LOGIC|0088| "notify_tokens":[REDACTED]
|
||||
ALERT_LOGIC|0089| "raw_issue_keys":sorted(cur),
|
||||
ALERT_LOGIC|0090| "code_counts":{k:int(v) for k,v in sorted(counts.items())},
|
||||
ALERT_LOGIC|0091| "migration_silent":bool(migrating)
|
||||
ALERT_LOGIC|0092|}
|
||||
ALERT_LOGIC|0093|tmp.write_text(json.dumps(payload,separators=(",",":"),sort_keys=True)+"\n",encoding="utf-8")
|
||||
ALERT_LOGIC|0094|os.chmod(tmp,0o600)
|
||||
ALERT_LOGIC|0095|os.replace(tmp,STATE)
|
||||
=== CANONICAL_SOURCE_CANDIDATES ===
|
||||
SOURCE_CANDIDATE|name=homelab-backup-audit|exact=false|sha256=1474c9ce58bbc1b3ae7437261d1bdb923232926fb2943c7c830253f8b284724c|path=/root/homelab-ops-cr-2026-1005/tasks/backup-v2-health-alerting-remediation-v1/assets/baseline/homelab-backup-audit
|
||||
SOURCE_CANDIDATE_TOTAL|name=homelab-backup-audit|count=1|exact_count=0
|
||||
SOURCE_CANDIDATE|name=homelab-alertmanager-backup-run|exact=true|sha256=2b92cd44fc5b43decba0ea0706c4c84e1912cdc6d7b9bcb10bbf9552c98dc3f9|path=/root/homelab-ops-cr-2026-1005/changes/CR-2026-1005/runtime/homelab-alertmanager-backup-run
|
||||
SOURCE_CANDIDATE|name=homelab-alertmanager-backup-run|exact=false|sha256=6deef30b4708a7e5b2ac0f0d98bb6b37b22d858afde994483b67baa18dffdc2a|path=/root/homelab-ops-cr-2026-1005/tasks/backup-v2-health-alerting-remediation-v1/assets/baseline/homelab-alertmanager-backup-run
|
||||
SOURCE_CANDIDATE_TOTAL|name=homelab-alertmanager-backup-run|count=2|exact_count=1
|
||||
=== SOURCE_RELEVANT_GREP ===
|
||||
changes/CR-2026-1005/runtime/homelab-backup-audit-v2:188:for lock in ["/run/homelab-backup-heavy.lock","/run/lock/homelab-app-cloud-quorum-queue.lock","/run/lock/homelab-vm-cloud-quorum-queue.lock","/run/lock/homelab-backup-dispatcher.lock"]:
|
||||
changes/CR-2026-1005/runtime/homelab-backup-audit-v2:192: if lrc!=0: issues.append({"severity":"HIGH","code":"LOCK_BUSY","logical_id":lock,"root_cause":"ACTIVE_OR_ORPHAN_LOCK"})
|
||||
changes/CR-2026-1005/runtime/homelab-backup-audit-v2:292:source={"local_head":None,"remote_head":None,"pushurl_disabled":False}
|
||||
changes/CR-2026-1005/runtime/homelab-backup-audit-v2:299: lrc,lo,_=run(["git","ls-remote",fo.strip(),"refs/heads/"+BRANCH],30)
|
||||
changes/CR-2026-1005/runtime/homelab-backup-audit-v2:300: if lrc==0 and lo.strip(): source["remote_head"]=lo.split()[0]
|
||||
changes/CR-2026-1005/runtime/homelab-backup-audit-v2:301:if source["remote_head"]!=source["local_head"]:
|
||||
changes/CR-2026-1005/runtime/homelab-backup-audit-v2:302: issues.append({"severity":"MEDIUM","code":"SOURCE_OFFHOST_MISSING","logical_id":"-","root_cause":"CR_BRANCH_SHA_NOT_EXACT"})
|
||||
changes/CR-2026-1005/runtime/homelab-backup-audit-v2:309:print(f"SUMMARY|AUDIT_STATUS=OK|BACKUP_STATUS={status}|ACTIVE_COUNT={len(active)}|ISSUE_COUNT={len(issues)}|CLOUD_A={'OK' if cloud.get('A',{}).get('readable') else 'BAD'}|CLOUD_B={'OK' if cloud.get('B',{}).get('readable') else 'BAD'}|CLOUD_B_DRILL={'OK' if cloudb_ok else 'BAD'}|DR_PROVEN={'YES' if isinstance(dr,dict) and dr.get('status')=='SEALED' else 'NO'}|SOURCE_OFFHOST={'OK' if source['remote_head']==source['local_head'] else 'PENDING'}")
|
||||
changes/CR-2026-1005/runtime/homelab-backup-retention-gc:14:HEAVY_LOCK=/run/homelab-backup-heavy.lock
|
||||
changes/CR-2026-1005/runtime/homelab-backup-retention-gc:424: exec 5>"$LOCK"; flock -n 5 || die GC_LOCK_BUSY 75
|
||||
changes/CR-2026-1005/runtime/homelab-backup-retention-gc:434: exec 5>"$LOCK"; flock -n 5 || { printf 'GC_SCHEDULED=SKIP|reason=GC_LOCK_BUSY\n'; return 0; }
|
||||
changes/CR-2026-1005/runtime/homelab-backup-retention-gc:444: if ! acquire_gc_locks; then write_health "$p1" true 0; printf 'GC_SCHEDULED=SKIP|reason=BACKUP_LOCK_BUSY|candidates=%s|open_transactions=%s\n' "$count" "$open_tx"; return 0; fi
|
||||
changes/CR-2026-1005/runtime/homelab-backup-v2-app-scheduled:6:HEAVY_LOCK=/run/homelab-backup-heavy.lock
|
||||
changes/CR-2026-1005/runtime/homelab-backup-v2-app-scheduled:19:exec 9>"$QUEUE_LOCK"; if ! flock -n 9; then printf 'APP_V2_SCHEDULED=SKIP|reason=APP_QUEUE_LOCK_BUSY\n'; exit 0; fi
|
||||
changes/CR-2026-1005/runtime/homelab-backup-v2-app-scheduled:20:exec 8>"$HEAVY_LOCK"; if ! flock -n 8; then printf 'APP_V2_SCHEDULED=SKIP|reason=GLOBAL_HEAVY_LOCK_BUSY\n'; exit 0; fi
|
||||
tasks/backup-v2-health-alerting-remediation-v1/assets/baseline/homelab-backup-audit:159:flock -n 9 || die AUDIT_LOCK_BUSY 75
|
||||
tasks/backup-v2-health-alerting-remediation-v1/assets/baseline/homelab-backup-audit:422:for lock in /run/homelab-backup-heavy.lock /run/lock/homelab-vm-cloud-quorum-queue.lock /run/lock/homelab-app-cloud-quorum-queue.lock /run/lock/homelab-backup-dispatcher.lock; do
|
||||
tasks/backup-v2-health-alerting-remediation-v1/assets/baseline/homelab-backup-retention-gc:14:HEAVY_LOCK=/run/homelab-backup-heavy.lock
|
||||
tasks/backup-v2-health-alerting-remediation-v1/assets/baseline/homelab-backup-retention-gc:424: exec 5>"$LOCK"; flock -n 5 || die GC_LOCK_BUSY 75
|
||||
tasks/backup-v2-health-alerting-remediation-v1/assets/baseline/homelab-backup-retention-gc:434: exec 5>"$LOCK"; flock -n 5 || { printf 'GC_SCHEDULED=SKIP|reason=GC_LOCK_BUSY\n'; return 0; }
|
||||
tasks/backup-v2-health-alerting-remediation-v1/assets/baseline/homelab-backup-retention-gc:444: if ! acquire_gc_locks; then write_health "$p1" true 0; printf 'GC_SCHEDULED=SKIP|reason=BACKUP_LOCK_BUSY|candidates=%s|open_transactions=%s\n' "$count" "$open_tx"; return 0; fi
|
||||
tasks/snikket-vm150-rebuild-v2/assets/vm150-runtime.py:60:BACKUP_LOCKS=['/run/lock/homelab-backup-dispatcher.lock','/run/lock/homelab-vm-cloud-quorum-queue.lock','/run/homelab-backup-heavy.lock']
|
||||
tasks/snikket-vm150-rebuild-v2/assets/vm150-runtime.py:506: f.close(); die('LOCK_BUSY_'+pathlib.Path(q).name,75)
|
||||
TASK_RESULT=PASS_BACKUP_ALERT_NOISE_ROOT_RCA_COLLECTED
|
||||
DECISION=PASS_BACKUP_ALERT_NOISE_ROOT_RCA_COLLECTED
|
||||
NEXT_GATE=READ_EXACT_1238_AND_APPLY_ONLY_PROVEN_ALERT_NOISE_FIX
|
||||
HOMELAB_RESULT_CONTRACT={"version":1,"command_id":"SUPPORT-260928-HOMELAB-BACKUP-ALERT-NOISE-ROOT-RCA-1238R1","status":"OK","changes_made":false,"rollback_started":false,"rollback_restored":null}
|
||||
CANON_AUDIT_SHA256=f7b936fa997bdefdd0c3c5d1d96e496888ebebc8ea85a60ed3bf5bcc9bf4962a
|
||||
CANON_ALERT_SHA256=2b92cd44fc5b43decba0ea0706c4c84e1912cdc6d7b9bcb10bbf9552c98dc3f9
|
||||
=== BUILD_PATCHED_AUDIT ===
|
||||
PATCH_BUILD_RC=0
|
||||
NEW_AUDIT_SHA256=1c2cd8a00859492c010ed35baf5eb67f4278544809de8c2563421a6f3e356ed3
|
||||
STATIC_EXPECTED_OWNER_CALL_COUNT=2
|
||||
STATIC_GIT_HOME_SET_COUNT=1
|
||||
TASK_RESULT=HOLD_BACKUP_AUDIT_NOISE_FIX
|
||||
DECISION=HOLD_1239_STATIC_PATCH_VERIFY_FAILED
|
||||
NEXT_GATE=FIX_ONLY_PATCH_BUILDER
|
||||
HOMELAB_RESULT_CONTRACT={"version":1,"command_id":"SUPPORT-260928-HOMELAB-BACKUP-AUDIT-NOISE-FIX-1239R1","status":"OK","changes_made":false,"rollback_started":false,"rollback_restored":null}
|
||||
|
||||
OUTPUT_END
|
||||
CHAT_OUTPUT_END
|
||||
|
||||
Reference in New Issue
Block a user