From 47c8b0153344abd7de107a8b9502d403b06844c6 Mon Sep 17 00:00:00 2001 From: homelab-runtime-publisher Date: Wed, 23 Sep 2026 12:08:42 +0000 Subject: [PATCH] runtime: publish SUPPORT-260923-SKLADCHIK-PREFLIGHT-LINE13-RCA-943R1 --- ...-SKLADCHIK-PREFLIGHT-LINE13-RCA-943R1.json | 38 ++ ...3-SKLADCHIK-PREFLIGHT-LINE13-RCA-943R1.txt | 349 ++++++++++++++ runtime/latest.json | 32 +- runtime/latest.txt | 428 ++++++++++++++---- 4 files changed, 731 insertions(+), 116 deletions(-) create mode 100644 runtime/history/SUPPORT-260923-SKLADCHIK-PREFLIGHT-LINE13-RCA-943R1.json create mode 100644 runtime/history/SUPPORT-260923-SKLADCHIK-PREFLIGHT-LINE13-RCA-943R1.txt diff --git a/runtime/history/SUPPORT-260923-SKLADCHIK-PREFLIGHT-LINE13-RCA-943R1.json b/runtime/history/SUPPORT-260923-SKLADCHIK-PREFLIGHT-LINE13-RCA-943R1.json new file mode 100644 index 00000000..f54056ab --- /dev/null +++ b/runtime/history/SUPPORT-260923-SKLADCHIK-PREFLIGHT-LINE13-RCA-943R1.json @@ -0,0 +1,38 @@ +{ + "schema_version": 1, + "channel": "homelab-runtime", + "command_id": "SUPPORT-260923-SKLADCHIK-PREFLIGHT-LINE13-RCA-943R1", + "status": "OK", + "rc": 0, + "host": "pve01", + "mode": "read-only", + "component": "skladchik-preflight-line13-rca", + "started_at_utc": "2026-09-23T12:08:41Z", + "finished_at_utc": "2026-09-23T12:08:41Z", + "reference_register_checked": true, + "reference_sha256": "5a3d8e5154c41cb582a4e0aca68090be1f0138918bf82131a948df326f9d8d66", + "error_register_checked": true, + "error_register_sha256": "3b09a553ec0f527ed3afeed4753f52a74ee3036045bbfb3c685e9f8af4ba7ba0", + "command_sha256": "dd3a88084befa0f31cc5f49ef676cb01eec2dfd507c1e1bfd993fba637e753d7", + "duplicate_failed_command_blocked": false, + "block_reason": null, + "execution_started": true, + "change_declared": false, + "result_contract_valid": true, + "result_contract_status": null, + "result_contract_error": null, + "command_rc": 0, + "changes_made": false, + "rollback_started": false, + "rollback_restored": null, + "mutation_outcome": "NO_MUTATION", + "sanitized": true, + "secrets_included": false, + "private_addresses_included": false, + "raw_evidence_retained_locally": true, + "raw_evidence_sha256": "2c42ff80da827c41e303d1369683a91c76fe254e8a5cc304eeb6038334a8dafa", + "sanitized_output_sha256": "17ace4d0e724d6b41d9460dad37dfe85e5b452f724ea763d0ba457d6ad255c77", + "output_truncated_in_json": false, + "full_sanitized_output_url": "https://git.gram1.ru/.well-known/homelab-runtime/latest.txt", + "output": "SKLADCHIK943_BEGIN=true\nCOMMAND_ID=SUPPORT-260923-SKLADCHIK-PREFLIGHT-LINE13-RCA-943R1\nMODE=read-only\nCOMPONENT=skladchik-preflight-line13-rca\nPREDECESSOR_942=FAILED_STAGE_PREFLIGHT;DETAIL_phase_preflight_line_13_rc_1\nMUTATION_BOUNDARY=NONE_READ_ONLY\nPHASE_PREFLIGHT_SOURCE_BEGIN\nPHASE_PREFLIGHT_SOURCE_PRESENT=true\nPHASE_PREFLIGHT_SOURCE_SHA256=f3e533ebd72ba2343a78797a8087c529c9fcfa2b3cabc45b3c0bd477356a8cc9\nPHASE_PREFLIGHT_SOURCE_BYTES=3076\nPHASE_PREFLIGHT_SOURCE_LINES=29\n0001:#!/usr/bin/env bash\n0002:set -Eeuo pipefail\n0003:on_err(){ local rc=$?; trap - ERR; printf 'ROOT_CAUSE=UNEXPECTED_PHASE_ERROR\\nROOT_CAUSE_COUNT=1\\nDETAIL=phase=preflight line=%s rc=%s\\n' \"${1:-0}\" \"$rc\"; exit \"$rc\"; }\n0004:trap 'on_err \"$LINENO\"' ERR\n0005:export SYSTEMD_PAGER=cat PAGER=cat SYSTEMD_LESS=FRXMK\n0006:TASK_ID=skladchik-reports-monitor-controlled-reauth-v1\n0007:EDGE=debian@[PRIVATE_IP]\n0008:CURRENT_EDGE_SHA=b13d846cf089d450e3d5ac9e85ad1e232c198930e466bb8b82f6983a4d48ce0e\n0009:TARGET_EDGE_SHA=10fe854274dca1a053cfd8c0155340bd689bf3f6555c70dbc14b890cbafb93fa\n0010:SSH=(-o BatchMode=yes -o ConnectTimeout=12 -o StrictHostKeyChecking=yes -o UserKnownHostsFile=[SENSITIVE_PATH]\n0011:test \"$(hostname -f)\" = pve01.gram1.ru\n0012:test -z \"$(git -C /srv/homelab-desired-state status --porcelain)\"\n0013:test \"$(sha256sum /usr/local/sbin/skladchik-reports-monitor-cookie-update | awk '{print $1}')\" = a3006723d902fb7735c593003a6c6755fa3d69683126ec5f53165131f754fd7a\n0014:test \"$(sha256sum /usr/local/sbin/skladchik-reports-monitor-backup | awk '{print $1}')\" = ad0652d1e154fca94a20bdf831b40b831fd0d862eb104ec3377f486e96270d0a\n0015:test \"$(sha256sum /usr/local/sbin/skladchik-reports-monitor-monthly-selftest-watch | awk '{print $1}')\" = dd2c7e26aaa6835fa4ac4365eade338eb7b19f4bfe7528179c59cf8039454180\n0016:test \"$(sha256sum /usr/local/sbin/skladchik-reports-monitor-status | awk '{print $1}')\" = f94ec75155f415b35517dade0bfba00926eba6088e0c2b5c96ae79daa5dc368f\n0017:test \"$(sha256sum /usr/local/sbin/skladchik-reports-monitor-supervisor | awk '{print $1}')\" = 69494a4c32fa632844cdb54aabbe302a14d478924efcb719c6cd3d594d102369\n0018:test \"$(sha256sum /srv/homelab-desired-state/sbin/skladchik-reports-monitor-backup | awk '{print $1}')\" = ad0652d1e154fca94a20bdf831b40b831fd0d862eb104ec3377f486e96270d0a\n0019:test \"$(sha256sum \"$HOMELAB_TASK_DIR/assets/edge/skladchik-reports-monitor-cookie-update-edge\" | awk '{print $1}')\" = \"$TARGET_EDGE_SHA\"\n0020:RESULT=\"$(ssh \"${SSH[@]}\" \"$EDGE\" sudo -n /bin/bash -s -- \"$CURRENT_EDGE_SHA\" < \"$HOMELAB_TASK_DIR/assets/edge-preflight.sh\")\"\n0021:grep -Fq 'EDGE_PREFLIGHT_STATUS=OK' <<<\"$RESULT\"\n0022:test \"$(systemctl is-enabled skladchik-reports-monitor-supervisor.timer)\" = enabled\n0023:test \"$(systemctl is-active skladchik-reports-monitor-supervisor.timer)\" = active\n0024:test \"$(systemctl show skladchik-reports-monitor-supervisor.service -p Result --value)\" = exit-code\n0025:grep -q '^STATUS=OK$' /var/lib/skladchik-reports-monitor-backup/health.txt\n0026:grep -q '^STATUS=OK$' /var/lib/skladchik-reports-monitor-backup/restore-check-health.txt\n0027:jq -n --arg status OK --arg current_edge_script_sha \"$CURRENT_EDGE_SHA\" --arg target_edge_script_sha \"$TARGET_EDGE_SHA\" '{status:$status,root_cause_count:0,production_changed:false,edge_monitor_status:\"AUTH_REQUIRED\",current_edge_script_sha256:$current_edge_script_sha,target_edge_script_sha256:$target_edge_script_sha,strict_hostkey:true,active_skladchik_sources_pinned:true}' > \"$HOMELAB_RUN_DIR/preflight.json\"\n0028:printf '%s\\n' \"$RESULT\"\n0029:printf 'PREFLIGHT_STATUS=OK\\nACTIVE_SKLADCHIK_SOURCE_HASHES=PINNED\\nEDGE_MUTATION_SCRIPT=VERSIONED\\nROOT_CAUSE_COUNT=0\\nPRODUCTION_CHANGED=NO\\nREAUTH_EXECUTED=NO\\n'\nPHASE_PREFLIGHT_SOURCE_END\nEDGE_PREFLIGHT_SOURCE_BEGIN\nEDGE_PREFLIGHT_SOURCE_PRESENT=true\nEDGE_PREFLIGHT_SOURCE_SHA256=77ebb2893962550f97d80bdfd2eb414f666578a272b6c35a3f4ca669df117878\nEDGE_PREFLIGHT_SOURCE_BYTES=1351\nEDGE_PREFLIGHT_SOURCE_LINES=29\n0001:#!/usr/bin/env bash\n0002:set -Eeuo pipefail\n0003:EXPECTED_SHA=\"${1:?expected script sha required}\"\n0004:SCRIPT=/usr/local/sbin/skladchik-reports-monitor-cookie-update-edge\n0005:HEALTH=/var/lib/skladchik-reports-monitor/health.txt\n0006:test -x \"$SCRIPT\"\n0007:test \"$(sha256sum \"$SCRIPT\" | awk '{print $1}')\" = \"$EXPECTED_SHA\"\n0008:test -s /etc/skladchik-reports-monitor/secrets.env\n0009:test -s /opt/skladchik-reports-monitor/monitor.py\n0010:test -s \"$HEALTH\"\n0011:STATUS=\"$(awk -F= '$1==\"STATUS\"{print $2;exit}' \"$HEALTH\")\"\n0012:test \"$STATUS\" = AUTH_REQUIRED\n0013:test \"$(systemctl is-enabled skladchik-reports-monitor.timer)\" = enabled\n0014:test \"$(systemctl is-active skladchik-reports-monitor.timer)\" = active\n0015:WAIT_SECONDS=90\n0016:DEADLINE=$((SECONDS+WAIT_SECONDS))\n0017:monitor_busy() {\n0018: systemctl is-active --quiet skladchik-reports-monitor.service ||\n0019: pgrep -f '/opt/skladchik-reports-monitor/monitor.py' >/dev/null ||\n0020: pgrep -f 'skladchik-reports-monitor-cookie-update-edge' >/dev/null\n0021:}\n0022:while monitor_busy; do\n0023: if (( SECONDS >= DEADLINE )); then\n0024: printf 'EDGE_PREFLIGHT_STATUS=BUSY\\nFAILED_PHASE=WAIT_MONITOR_QUIESCENCE\\nWAIT_SECONDS=%s\\n' \"$WAIT_SECONDS\"\n0025: exit 75\n0026: fi\n0027: sleep 2\n0028:done\n0029:printf 'EDGE_PREFLIGHT_STATUS=OK\\nEDGE_MONITOR_STATUS=%s\\nEDGE_SCRIPT_SHA256=%s\\nEDGE_TIMER_ENABLED=YES\\nEDGE_TIMER_ACTIVE=YES\\nMONITOR_QUIESCENCE=OK\\nWAIT_LIMIT_SECONDS=%s\\n' \"$STATUS\" \"$EXPECTED_SHA\" \"$WAIT_SECONDS\"\nEDGE_PREFLIGHT_SOURCE_END\nFAILED_LINE_CONTEXT_BEGIN\nPHASE_LINE_0009=TARGET_EDGE_SHA=10fe854274dca1a053cfd8c0155340bd689bf3f6555c70dbc14b890cbafb93fa\nPHASE_LINE_0010=SSH=(-o BatchMode=yes -o ConnectTimeout=12 -o StrictHostKeyChecking=yes -o UserKnownHostsFile=[SENSITIVE_PATH]\nPHASE_LINE_0011=test \"$(hostname -f)\" = pve01.gram1.ru\nPHASE_LINE_0012=test -z \"$(git -C /srv/homelab-desired-state status --porcelain)\"\nPHASE_LINE_0013=test \"$(sha256sum /usr/local/sbin/skladchik-reports-monitor-cookie-update | awk '{print $1}')\" = a3006723d902fb7735c593003a6c6755fa3d69683126ec5f53165131f754fd7a\nPHASE_LINE_0014=test \"$(sha256sum /usr/local/sbin/skladchik-reports-monitor-backup | awk '{print $1}')\" = ad0652d1e154fca94a20bdf831b40b831fd0d862eb104ec3377f486e96270d0a\nPHASE_LINE_0015=test \"$(sha256sum /usr/local/sbin/skladchik-reports-monitor-monthly-selftest-watch | awk '{print $1}')\" = dd2c7e26aaa6835fa4ac4365eade338eb7b19f4bfe7528179c59cf8039454180\nPHASE_LINE_0016=test \"$(sha256sum /usr/local/sbin/skladchik-reports-monitor-status | awk '{print $1}')\" = f94ec75155f415b35517dade0bfba00926eba6088e0c2b5c96ae79daa5dc368f\nPHASE_LINE_0017=test \"$(sha256sum /usr/local/sbin/skladchik-reports-monitor-supervisor | awk '{print $1}')\" = 69494a4c32fa632844cdb54aabbe302a14d478924efcb719c6cd3d594d102369\nEDGE_LINE_0009=test -s /opt/skladchik-reports-monitor/monitor.py\nEDGE_LINE_0010=test -s \"$HEALTH\"\nEDGE_LINE_0011=STATUS=\"$(awk -F= '$1==\"STATUS\"{print $2;exit}' \"$HEALTH\")\"\nEDGE_LINE_0012=test \"$STATUS\" = AUTH_REQUIRED\nEDGE_LINE_0013=test \"$(systemctl is-enabled skladchik-reports-monitor.timer)\" = enabled\nEDGE_LINE_0014=test \"$(systemctl is-active skladchik-reports-monitor.timer)\" = active\nEDGE_LINE_0015=WAIT_SECONDS=90\nEDGE_LINE_0016=DEADLINE=$((SECONDS+WAIT_SECONDS))\nEDGE_LINE_0017=monitor_busy() {\nFAILED_LINE_CONTEXT_END\nRUN_EVIDENCE_BEGIN\nRUN_FILE=stdout.log|present=true|bytes=89\nstdout.log|ROOT_CAUSE=UNEXPECTED_PHASE_ERROR\nstdout.log|ROOT_CAUSE_COUNT=1\nstdout.log|DETAIL=phase=preflight line=13 rc=1\nRUN_FILE=stderr.log|present=true|bytes=0\nRUN_FILE=preflight.json|present=false|bytes=0\nRUN_FILE=result.json|present=true|bytes=592\nresult.json|{\nresult.json| \"run_id\": \"20260923T120316Z-986653-preflight\",\nresult.json| \"task_id\": \"skladchik-reports-monitor-controlled-reauth-v1\",\nresult.json| \"phase\": \"preflight\",\nresult.json| \"mutation_scope\": \"controlled_apply\",\nresult.json| \"production_change\": true,\nresult.json| \"started_at\": \"2026-09-23T12:03:16Z\",\nresult.json| \"finished_at\": \"2026-09-23T12:03:16Z\",\nresult.json| \"status\": \"FAILED\",\nresult.json| \"rc\": 1,\nresult.json| \"stdout\": \"/var/lib/homelab-admin/runs/skladchik-reports-monitor-controlled-reauth-v1/20260923T120316Z-986653-preflight/stdout.log\",\nresult.json| \"stderr\": \"/var/lib/homelab-admin/runs/skladchik-reports-monitor-controlled-reauth-v1/20260923T120316Z-986653-preflight/stderr.log\"\nresult.json|}\nRUN_FILE=task.json|present=true|bytes=13579\ntask.json|{\ntask.json| \"schema_version\": 2,\ntask.json| \"task_id\": \"skladchik-reports-monitor-controlled-reauth-v1\",\ntask.json| \"target_host\": \"pve01.gram1.ru\",\ntask.json| \"mutation_scope\": \"controlled_apply\",\ntask.json| \"production_change\": true,\ntask.json| \"failure_mode\": \"fail_fast\",\ntask.json| \"guards\": {\ntask.json| \"desired_state_clean\": true,\ntask.json| \"failed_units_max\": 4,\ntask.json| \"active_backup_restore_max\": 0\ntask.json| },\ntask.json| \"reads\": [\ntask.json| \"[SENSITIVE_PATH]\ntask.json| \"/var/lib/skladchik-reports-monitor-backup/health.txt\",\ntask.json| \"/var/lib/skladchik-reports-monitor-backup/restore-check-health.txt\",\ntask.json| \"systemd:skladchik-reports-monitor-supervisor.service\",\ntask.json| \"systemd:skladchik-reports-monitor-supervisor.timer\",\ntask.json| \"ssh:debian@[PRIVATE_IP]:/usr/local/sbin/skladchik-reports-monitor-cookie-update-edge\",\ntask.json| \"ssh:debian@[PRIVATE_IP]:/etc/skladchik-reports-monitor/secrets.env\",\ntask.json| \"ssh:debian@[PRIVATE_IP]:/var/lib/skladchik-reports-monitor/cookies.jar\",\ntask.json| \"ssh:debian@[PRIVATE_IP]:/var/lib/skladchik-reports-monitor/health.txt\",\ntask.json| \"/usr/local/sbin/skladchik-reports-monitor-cookie-update\",\ntask.json| \"/usr/local/sbin/skladchik-reports-monitor-backup\",\ntask.json| \"/usr/local/sbin/skladchik-reports-monitor-monthly-selftest-watch\",\ntask.json| \"/usr/local/sbin/skladchik-reports-monitor-status\",\ntask.json| \"/usr/local/sbin/skladchik-reports-monitor-supervisor\",\ntask.json| \"/srv/homelab-desired-state/sbin/skladchik-reports-monitor-backup\",\ntask.json| \"/var/lib/skladchik-reports-monitor-supervisor\"\ntask.json| ],\ntask.json| \"writes\": [\ntask.json| \"/var/lib/homelab-admin/seals/skladchik-reports-monitor-controlled-reauth-v1.json\",\ntask.json| \"systemd:skladchik-reports-monitor-supervisor.service\",\ntask.json| \"ssh:debian@[PRIVATE_IP]:/etc/skladchik-reports-monitor/secrets.env\",\ntask.json| \"ssh:debian@[PRIVATE_IP]:/var/lib/skladchik-reports-monitor/cookies.jar\",\ntask.json| \"ssh:debian@[PRIVATE_IP]:/var/lib/skladchik-reports-monitor/health.txt\",\ntask.json| \"ssh:debian@[PRIVATE_IP]:/var/lib/homelab-admin/rollback/skladchik-reports-monitor-controlled-reauth-v1\",\ntask.json| \"$HOMELAB_RUN_DIR/preflight.json\",\ntask.json| \"$HOMELAB_RUN_DIR/prepare.json\",\ntask.json| \"$HOMELAB_RUN_DIR/dry-run.json\",\ntask.json| \"$HOMELAB_RUN_DIR/apply.json\",\ntask.json| \"$HOMELAB_RUN_DIR/verify.json\",\ntask.json| \"$HOMELAB_RUN_DIR/rollback.json\",\ntask.json| \"$HOMELAB_RUN_DIR/seal.json\",\ntask.json| \"/usr/local/sbin/skladchik-reports-monitor-cookie-update\",\ntask.json| \"/usr/local/sbin/skladchik-reports-monitor-backup\",\ntask.json| \"/usr/local/sbin/skladchik-reports-monitor-monthly-selftest-watch\",\ntask.json| \"/usr/local/sbin/skladchik-reports-monitor-status\",\ntask.json| \"/usr/local/sbin/skladchik-reports-monitor-supervisor\",\ntask.json| \"/srv/homelab-desired-state/sbin/skladchik-reports-monitor-cookie-update\",\ntask.json| \"/srv/homelab-desired-state/sbin/skladchik-reports-monitor-backup\",\ntask.json| \"/srv/homelab-desired-state/sbin/skladchik-reports-monitor-monthly-selftest-watch\",\ntask.json| \"/srv/homelab-desired-state/sbin/skladchik-reports-monitor-status\",\ntask.json| \"/srv/homelab-desired-state/sbin/skladchik-reports-monitor-supervisor\",\ntask.json| \"/mnt/staging/homelab-admin-rollback/skladchik-reports-monitor-controlled-reauth-v1\",\ntask.json| \"ssh:debian@[PRIVATE_IP]:/usr/local/sbin/skladchik-reports-monitor-cookie-update-edge\"\ntask.json| ],\ntask.json| \"mutates\": [\ntask.json| \"systemd:skladchik-reports-monitor-supervisor.service\",\ntask.json| \"ssh:debian@[PRIVATE_IP]:/etc/skladchik-reports-monitor/secrets.env\",\ntask.json| \"ssh:debian@[PRIVATE_IP]:/var/lib/skladchik-reports-monitor/cookies.jar\",\ntask.json| \"ssh:debian@[PRIVATE_IP]:systemd:skladchik-reports-monitor.service\",\ntask.json| \"ssh:debian@[PRIVATE_IP]:systemd:skladchik-reports-monitor.timer\",\ntask.json| \"/usr/local/sbin/skladchik-reports-monitor-cookie-update\",\ntask.json| \"/usr/local/sbin/skladchik-reports-monitor-backup\",\ntask.json| \"/usr/local/sbin/skladchik-reports-monitor-monthly-selftest-watch\",\ntask.json| \"/usr/local/sbin/skladchik-reports-monitor-status\",\ntask.json| \"/usr/local/sbin/skladchik-reports-monitor-supervisor\",\ntask.json| \"git:/srv/homelab-desired-state\",\ntask.json| \"ssh:debian@[PRIVATE_IP]:/usr/local/sbin/skladchik-reports-monitor-cookie-update-edge\"\ntask.json| ],\ntask.json| \"expected_outputs\": [\ntask.json| {\ntask.json| \"path\": \"$HOMELAB_RUN_DIR/preflight.json\",\ntask.json| \"producer\": \"phases/preflight.sh\",\ntask.json| \"verification\": \"strict host-key access, the pinned live predecessor edge script, the versioned target edge script, AUTH_REQUIRED and bounded monitor quiescence are confirmed; evidence records both SHAs\"\ntask.json| },\ntask.json| {\ntask.json| \"path\": \"$HOMELAB_RUN_DIR/prepare.json\",\ntask.json| \"producer\": \"phases/prepare.sh\",\ntask.json| \"verification\": \"root-only local and edge rollback snapshots have self-contained SHA256 manifests\"\ntask.json| },\ntask.json| {\ntask.json| \"path\": \"$HOMELAB_RUN_DIR/dry-run.json\",\ntask.json| \"producer\": \"phases/dry-run.sh\",\ntask.json| \"verification\": \"rollback manifests validate, the pre-apply live edge script remains the pinned predecessor, the versioned deployment asset matches the pinned target SHA256, and evidence records both SHAs\"\ntask.json| },\ntask.json| {\ntask.json| \"path\": \"$HOMELAB_RUN_DIR/apply.json\",\ntask.json| \"producer\": \"phases/apply.sh\",\ntask.json| \"verification\": \"apply replaces the pinned predecessor edge script with the pinned target script, quiesces the timer, performs one controlled reauthentication and restores or enables the timer\"\ntask.json| },\ntask.json| {\ntask.json| \"path\": \"$HOMELAB_RUN_DIR/verify.json\",\ntask.json| \"producer\": \"phases/verify.sh\",\ntask.json| \"verification\": \"edge monitor, counts, timer, supervisor, backup and restore are OK\"\ntask.json| },\ntask.json| {\ntask.json| \"path\": \"$HOMELAB_RUN_DIR/rollback.json\",\ntask.json| \"producer\": \"phases/rollback.sh\",\ntask.json| \"verification\": \"previous auth artifacts and timer state are restored\"\ntask.json| },\ntask.json| {\ntask.json| \"path\": \"$HOMELAB_RUN_DIR/seal.json\",\ntask.json| \"producer\": \"phases/seal.sh\",\ntask.json| \"verification\": \"standard seal exists and rollback snapshot is removed\"\ntask.json| }\ntask.json| ],\ntask.json| \"runtime_matrix\": [\ntask.json| {\ntask.json| \"host\": \"pve01.gram1.ru\",\ntask.json| \"runtime\": \"bash,jq,git,systemd,ssh\",\ntask.json| \"compile\": \"bash -n; jq -e\"\ntask.json| },\ntask.json| {\ntask.json| \"host\": \"edge\",\ntask.json| \"runtime\": \"bash,python3,systemd,sha256sum\",\ntask.json| \"compile\": \"bash -n\"\ntask.json| }\ntask.json| ],\ntask.json| \"rollback\": {\ntask.json| \"command\": \"HOMELAB_CONTROLLED_APPLY_CONFIRM=skladchik-reports-monitor-controlled-reauth-v1 /opt/homelab-admin/bin/homelab-admin run TASK_DIR rollback\",\ntask.json| \"verification\": \"self-contained local and edge manifests validate; prior auth artifacts, scripts, desired-state HEAD, supervisor state and timer state are restorable\"\ntask.json| },\ntask.json| \"no_change_assertions\": [\ntask.json| \"vm160_backup_job\",\ntask.json| \"cluster_backup_jobs\",\ntask.json| \"unrelated_guests\",\ntask.json| \"network\",\ntask.json| \"dns\",\ntask.json| \"known_hosts\",\ntask.json| \"mail_cloud\",\ntask.json| \"gitea\",\ntask.json| \"backup_and_restore_artifacts\"\ntask.json| ],\ntask.json| \"source_of_truth\": {\ntask.json| \"change_id\": \"CR-2026-0012\",\ntask.json| \"incident\": \"Skladchik monitor AUTH_REQUIRED with missing complaint counts\",\ntask.json| \"edge_script_sha256\": \"10fe854274dca1a053cfd8c0155340bd689bf3f6555c70dbc14b890cbafb93fa\",\ntask.json| \"edge_script_path\": \"/usr/local/sbin/skladchik-reports-monitor-cookie-update-edge\",\ntask.json| \"unsafe_legacy_wrapper_path\": \"/usr/local/sbin/skladchik-reports-monitor-cookie-update\",\ntask.json| \"unsafe_legacy_wrapper_reason\": \"legacy host-key auto-acceptance\",\ntask.json| \"strict_hostkey_required\": true,\ntask.json| \"hidden_interactive_prompt_required\": true,\ntask.json| \"cookie_content_must_not_enter_git_or_logs\": true,\ntask.json| \"backup_status_before\": \"OK\",\ntask.json| \"restore_status_before\": \"OK\",\ntask.json| \"regression_lock\": \"REG-024-SKLADCHIK-REAUTH-STRICT-HOSTKEY\",\ntask.json| \"active_pve01_script_count\": 5,\ntask.json| \"active_pve01_original_sha256\": {\ntask.json| \"skladchik-reports-monitor-cookie-update\": \"a3006723d902fb7735c593003a6c6755fa3d69683126ec5f53165131f754fd7a\",\ntask.json| \"skladchik-reports-monitor-backup\": \"ad0652d1e154fca94a20bdf831b40b831fd0d862eb104ec3377f486e96270d0a\",\ntask.json| \"skladchik-reports-monitor-monthly-selftest-watch\": \"dd2c7e26aaa6835fa4ac4365eade338eb7b19f4bfe7528179c59cf8039454180\",\ntask.json| \"skladchik-reports-monitor-status\": \"f94ec75155f415b35517dade0bfba00926eba6088e0c2b5c96ae79daa5dc368f\",\ntask.json| \"skladchik-reports-monitor-supervisor\": \"69494a4c32fa632844cdb54aabbe302a14d478924efcb719c6cd3d594d102369\"\ntask.json| },\ntask.json| \"active_pve01_deployed_sha256\": {\ntask.json| \"skladchik-reports-monitor-cookie-update\": \"ea2d89ce67893a6d89b29610e4d46e207932ecc157173454585f67890f58810c\",\ntask.json| \"skladchik-reports-monitor-backup\": \"1912f8f0559637563972bb8ac984fb0036694092056ae53968c02cc3b67855f3\",\ntask.json| \"skladchik-reports-monitor-monthly-selftest-watch\": \"48c6ca01d2430d14c6af797c079640815ba1352a98466d4014ecbb80a27eaada\",\ntask.json| \"skladchik-reports-monitor-status\": \"f902a61109efbac5918360f2235c2523bd10704ac484a4e8591e8a1c490faf5a\",\ntask.json| \"skladchik-reports-monitor-supervisor\": \"35c68ae0929ff3795be2bde40de334fc8e26413a3d8a620a137be90d1f2c164e\"\ntask.json| },\ntask.json| \"edge_mutation_script_versioned\": true,\ntask.json| \"desired_state_script_count_after_apply\": 5,\ntask.json| \"all_active_skladchik_ssh_requires_strict_known_hosts\": true,\ntask.json| \"unsafe_active_hostkey_markers_after_verify\": 0,\ntask.json| \"pve01_assets_checked_by_repository_tests\": true,\ntask.json| \"regression_locks\": [\ntask.json| \"REG-024-SKLADCHIK-REAUTH-STRICT-HOSTKEY\",\ntask.json| \"REG-025-EXACT-SHA256-LENGTH-VALIDATION\",\ntask.json| \"REG-026-ACTIVE-SKLADCHIK-STRICT-HOSTKEY-COVERAGE\",\ntask.json| \"REG-027-REMOTE-MUTATION-SCRIPT-MUST-BE-VERSIONED\",\ntask.json| \"REG-028-FORBIDDEN-TOKEN-IN-NEGATIVE-ASSERTION\",\ntask.json| \"REG-029-LEGACY-WRAPPER-NOT-COPIED-VERBATIM\",\ntask.json| \"REG-030-BINARY-ARTIFACT-TEXT-SCAN\",\ntask.json| \"REG-031-SEMANTIC-TEST-LOOP-EXPANSION-MISMATCH\",\ntask.json| \"REG-032-OVERBROAD-INLINE-PYTHON-TEST\",\ntask.json| \"REG-033-INSTALLED-SOURCE-MUST-BE-DECLARED\",\ntask.json| \"REG-034-ROLLBACK-STAGE-PARENT-MODE\",\nRUN_EVIDENCE_END\nCURRENT_FACTS_BEGIN\nEDGE_SCRIPT_LOCAL_EXISTS=true\nEDGE_SCRIPT_LOCAL_META=mode=700|uid=0|gid=0|size=5247\nEDGE_SCRIPT_LOCAL_SHA256=10fe854274dca1a053cfd8c0155340bd689bf3f6555c70dbc14b890cbafb93fa\nLOCAL_HEALTH_EXISTS=false\nSECRETS_ENV_EXISTS=false\nMONITOR_PY_EXISTS=false\nUNIT=skladchik-reports-monitor.service|active=inactive|enabled=not-found\nUNIT=skladchik-reports-monitor.timer|active=inactive|enabled=not-found\nUNIT=skladchik-reports-monitor-supervisor.service|active=failed|enabled=static\nUNIT=skladchik-reports-monitor-supervisor.timer|active=active|enabled=enabled\nCURRENT_FACTS_END\nRCA_CAPTURE_COMPLETE=true\nCHANGES_MADE_BY_943=false\nROLLBACK_STARTED=false\nROLLBACK_RESTORED=null\nHOMELAB_RESULT_CONTRACT={\"version\":1,\"command_id\":\"SUPPORT-260923-SKLADCHIK-PREFLIGHT-LINE13-RCA-943R1\",\"status\":\"OK\",\"changes_made\":false,\"rollback_started\":false,\"rollback_restored\":null}\nSKLADCHIK943_END=true\n" +} diff --git a/runtime/history/SUPPORT-260923-SKLADCHIK-PREFLIGHT-LINE13-RCA-943R1.txt b/runtime/history/SUPPORT-260923-SKLADCHIK-PREFLIGHT-LINE13-RCA-943R1.txt new file mode 100644 index 00000000..6893533f --- /dev/null +++ b/runtime/history/SUPPORT-260923-SKLADCHIK-PREFLIGHT-LINE13-RCA-943R1.txt @@ -0,0 +1,349 @@ +CHAT_OUTPUT_BEGIN +COMMAND_ID=SUPPORT-260923-SKLADCHIK-PREFLIGHT-LINE13-RCA-943R1 +STATUS=OK +RC=0 +HOST=pve01 +MODE=read-only +COMPONENT=skladchik-preflight-line13-rca +REFERENCE_REGISTER_CHECK=OK +REFERENCE_SHA256=5a3d8e5154c41cb582a4e0aca68090be1f0138918bf82131a948df326f9d8d66 +ERROR_REGISTER_CHECK=OK +ERROR_REGISTER_SHA256=3b09a553ec0f527ed3afeed4753f52a74ee3036045bbfb3c685e9f8af4ba7ba0 +COMMAND_SHA256=dd3a88084befa0f31cc5f49ef676cb01eec2dfd507c1e1bfd993fba637e753d7 +DUPLICATE_FAILED_COMMAND_BLOCKED=false +EXECUTION_STARTED=true +CHANGE_DECLARED=false +RESULT_CONTRACT_VALID=true +RESULT_CONTRACT_STATUS=NOT_APPLICABLE +RESULT_CONTRACT_ERROR=NONE +COMMAND_RC=0 +CHANGES_MADE=false +ROLLBACK_STARTED=false +ROLLBACK_RESTORED=null +MUTATION_OUTCOME=NO_MUTATION +SANITIZED=yes +SECRETS_INCLUDED=no +PRIVATE_ADDRESSES_INCLUDED=no +RAW_EVIDENCE_SHA256=2c42ff80da827c41e303d1369683a91c76fe254e8a5cc304eeb6038334a8dafa +SANITIZED_OUTPUT_SHA256=17ace4d0e724d6b41d9460dad37dfe85e5b452f724ea763d0ba457d6ad255c77 +OUTPUT_BEGIN +SKLADCHIK943_BEGIN=true +COMMAND_ID=SUPPORT-260923-SKLADCHIK-PREFLIGHT-LINE13-RCA-943R1 +MODE=read-only +COMPONENT=skladchik-preflight-line13-rca +PREDECESSOR_942=FAILED_STAGE_PREFLIGHT;DETAIL_phase_preflight_line_13_rc_1 +MUTATION_BOUNDARY=NONE_READ_ONLY +PHASE_PREFLIGHT_SOURCE_BEGIN +PHASE_PREFLIGHT_SOURCE_PRESENT=true +PHASE_PREFLIGHT_SOURCE_SHA256=f3e533ebd72ba2343a78797a8087c529c9fcfa2b3cabc45b3c0bd477356a8cc9 +PHASE_PREFLIGHT_SOURCE_BYTES=3076 +PHASE_PREFLIGHT_SOURCE_LINES=29 +0001:#!/usr/bin/env bash +0002:set -Eeuo pipefail +0003:on_err(){ local rc=$?; trap - ERR; printf 'ROOT_CAUSE=UNEXPECTED_PHASE_ERROR\nROOT_CAUSE_COUNT=1\nDETAIL=phase=preflight line=%s rc=%s\n' "${1:-0}" "$rc"; exit "$rc"; } +0004:trap 'on_err "$LINENO"' ERR +0005:export SYSTEMD_PAGER=cat PAGER=cat SYSTEMD_LESS=FRXMK +0006:TASK_ID=skladchik-reports-monitor-controlled-reauth-v1 +0007:EDGE=debian@[PRIVATE_IP] +0008:CURRENT_EDGE_SHA=b13d846cf089d450e3d5ac9e85ad1e232c198930e466bb8b82f6983a4d48ce0e +0009:TARGET_EDGE_SHA=10fe854274dca1a053cfd8c0155340bd689bf3f6555c70dbc14b890cbafb93fa +0010:SSH=(-o BatchMode=yes -o ConnectTimeout=12 -o StrictHostKeyChecking=yes -o UserKnownHostsFile=[SENSITIVE_PATH] +0011:test "$(hostname -f)" = pve01.gram1.ru +0012:test -z "$(git -C /srv/homelab-desired-state status --porcelain)" +0013:test "$(sha256sum /usr/local/sbin/skladchik-reports-monitor-cookie-update | awk '{print $1}')" = a3006723d902fb7735c593003a6c6755fa3d69683126ec5f53165131f754fd7a +0014:test "$(sha256sum /usr/local/sbin/skladchik-reports-monitor-backup | awk '{print $1}')" = ad0652d1e154fca94a20bdf831b40b831fd0d862eb104ec3377f486e96270d0a +0015:test "$(sha256sum /usr/local/sbin/skladchik-reports-monitor-monthly-selftest-watch | awk '{print $1}')" = dd2c7e26aaa6835fa4ac4365eade338eb7b19f4bfe7528179c59cf8039454180 +0016:test "$(sha256sum /usr/local/sbin/skladchik-reports-monitor-status | awk '{print $1}')" = f94ec75155f415b35517dade0bfba00926eba6088e0c2b5c96ae79daa5dc368f +0017:test "$(sha256sum /usr/local/sbin/skladchik-reports-monitor-supervisor | awk '{print $1}')" = 69494a4c32fa632844cdb54aabbe302a14d478924efcb719c6cd3d594d102369 +0018:test "$(sha256sum /srv/homelab-desired-state/sbin/skladchik-reports-monitor-backup | awk '{print $1}')" = ad0652d1e154fca94a20bdf831b40b831fd0d862eb104ec3377f486e96270d0a +0019:test "$(sha256sum "$HOMELAB_TASK_DIR/assets/edge/skladchik-reports-monitor-cookie-update-edge" | awk '{print $1}')" = "$TARGET_EDGE_SHA" +0020:RESULT="$(ssh "${SSH[@]}" "$EDGE" sudo -n /bin/bash -s -- "$CURRENT_EDGE_SHA" < "$HOMELAB_TASK_DIR/assets/edge-preflight.sh")" +0021:grep -Fq 'EDGE_PREFLIGHT_STATUS=OK' <<<"$RESULT" +0022:test "$(systemctl is-enabled skladchik-reports-monitor-supervisor.timer)" = enabled +0023:test "$(systemctl is-active skladchik-reports-monitor-supervisor.timer)" = active +0024:test "$(systemctl show skladchik-reports-monitor-supervisor.service -p Result --value)" = exit-code +0025:grep -q '^STATUS=OK$' /var/lib/skladchik-reports-monitor-backup/health.txt +0026:grep -q '^STATUS=OK$' /var/lib/skladchik-reports-monitor-backup/restore-check-health.txt +0027:jq -n --arg status OK --arg current_edge_script_sha "$CURRENT_EDGE_SHA" --arg target_edge_script_sha "$TARGET_EDGE_SHA" '{status:$status,root_cause_count:0,production_changed:false,edge_monitor_status:"AUTH_REQUIRED",current_edge_script_sha256:$current_edge_script_sha,target_edge_script_sha256:$target_edge_script_sha,strict_hostkey:true,active_skladchik_sources_pinned:true}' > "$HOMELAB_RUN_DIR/preflight.json" +0028:printf '%s\n' "$RESULT" +0029:printf 'PREFLIGHT_STATUS=OK\nACTIVE_SKLADCHIK_SOURCE_HASHES=PINNED\nEDGE_MUTATION_SCRIPT=VERSIONED\nROOT_CAUSE_COUNT=0\nPRODUCTION_CHANGED=NO\nREAUTH_EXECUTED=NO\n' +PHASE_PREFLIGHT_SOURCE_END +EDGE_PREFLIGHT_SOURCE_BEGIN +EDGE_PREFLIGHT_SOURCE_PRESENT=true +EDGE_PREFLIGHT_SOURCE_SHA256=77ebb2893962550f97d80bdfd2eb414f666578a272b6c35a3f4ca669df117878 +EDGE_PREFLIGHT_SOURCE_BYTES=1351 +EDGE_PREFLIGHT_SOURCE_LINES=29 +0001:#!/usr/bin/env bash +0002:set -Eeuo pipefail +0003:EXPECTED_SHA="${1:?expected script sha required}" +0004:SCRIPT=/usr/local/sbin/skladchik-reports-monitor-cookie-update-edge +0005:HEALTH=/var/lib/skladchik-reports-monitor/health.txt +0006:test -x "$SCRIPT" +0007:test "$(sha256sum "$SCRIPT" | awk '{print $1}')" = "$EXPECTED_SHA" +0008:test -s /etc/skladchik-reports-monitor/secrets.env +0009:test -s /opt/skladchik-reports-monitor/monitor.py +0010:test -s "$HEALTH" +0011:STATUS="$(awk -F= '$1=="STATUS"{print $2;exit}' "$HEALTH")" +0012:test "$STATUS" = AUTH_REQUIRED +0013:test "$(systemctl is-enabled skladchik-reports-monitor.timer)" = enabled +0014:test "$(systemctl is-active skladchik-reports-monitor.timer)" = active +0015:WAIT_SECONDS=90 +0016:DEADLINE=$((SECONDS+WAIT_SECONDS)) +0017:monitor_busy() { +0018: systemctl is-active --quiet skladchik-reports-monitor.service || +0019: pgrep -f '/opt/skladchik-reports-monitor/monitor.py' >/dev/null || +0020: pgrep -f 'skladchik-reports-monitor-cookie-update-edge' >/dev/null +0021:} +0022:while monitor_busy; do +0023: if (( SECONDS >= DEADLINE )); then +0024: printf 'EDGE_PREFLIGHT_STATUS=BUSY\nFAILED_PHASE=WAIT_MONITOR_QUIESCENCE\nWAIT_SECONDS=%s\n' "$WAIT_SECONDS" +0025: exit 75 +0026: fi +0027: sleep 2 +0028:done +0029:printf 'EDGE_PREFLIGHT_STATUS=OK\nEDGE_MONITOR_STATUS=%s\nEDGE_SCRIPT_SHA256=%s\nEDGE_TIMER_ENABLED=YES\nEDGE_TIMER_ACTIVE=YES\nMONITOR_QUIESCENCE=OK\nWAIT_LIMIT_SECONDS=%s\n' "$STATUS" "$EXPECTED_SHA" "$WAIT_SECONDS" +EDGE_PREFLIGHT_SOURCE_END +FAILED_LINE_CONTEXT_BEGIN +PHASE_LINE_0009=TARGET_EDGE_SHA=10fe854274dca1a053cfd8c0155340bd689bf3f6555c70dbc14b890cbafb93fa +PHASE_LINE_0010=SSH=(-o BatchMode=yes -o ConnectTimeout=12 -o StrictHostKeyChecking=yes -o UserKnownHostsFile=[SENSITIVE_PATH] +PHASE_LINE_0011=test "$(hostname -f)" = pve01.gram1.ru +PHASE_LINE_0012=test -z "$(git -C /srv/homelab-desired-state status --porcelain)" +PHASE_LINE_0013=test "$(sha256sum /usr/local/sbin/skladchik-reports-monitor-cookie-update | awk '{print $1}')" = a3006723d902fb7735c593003a6c6755fa3d69683126ec5f53165131f754fd7a +PHASE_LINE_0014=test "$(sha256sum /usr/local/sbin/skladchik-reports-monitor-backup | awk '{print $1}')" = ad0652d1e154fca94a20bdf831b40b831fd0d862eb104ec3377f486e96270d0a +PHASE_LINE_0015=test "$(sha256sum /usr/local/sbin/skladchik-reports-monitor-monthly-selftest-watch | awk '{print $1}')" = dd2c7e26aaa6835fa4ac4365eade338eb7b19f4bfe7528179c59cf8039454180 +PHASE_LINE_0016=test "$(sha256sum /usr/local/sbin/skladchik-reports-monitor-status | awk '{print $1}')" = f94ec75155f415b35517dade0bfba00926eba6088e0c2b5c96ae79daa5dc368f +PHASE_LINE_0017=test "$(sha256sum /usr/local/sbin/skladchik-reports-monitor-supervisor | awk '{print $1}')" = 69494a4c32fa632844cdb54aabbe302a14d478924efcb719c6cd3d594d102369 +EDGE_LINE_0009=test -s /opt/skladchik-reports-monitor/monitor.py +EDGE_LINE_0010=test -s "$HEALTH" +EDGE_LINE_0011=STATUS="$(awk -F= '$1=="STATUS"{print $2;exit}' "$HEALTH")" +EDGE_LINE_0012=test "$STATUS" = AUTH_REQUIRED +EDGE_LINE_0013=test "$(systemctl is-enabled skladchik-reports-monitor.timer)" = enabled +EDGE_LINE_0014=test "$(systemctl is-active skladchik-reports-monitor.timer)" = active +EDGE_LINE_0015=WAIT_SECONDS=90 +EDGE_LINE_0016=DEADLINE=$((SECONDS+WAIT_SECONDS)) +EDGE_LINE_0017=monitor_busy() { +FAILED_LINE_CONTEXT_END +RUN_EVIDENCE_BEGIN +RUN_FILE=stdout.log|present=true|bytes=89 +stdout.log|ROOT_CAUSE=UNEXPECTED_PHASE_ERROR +stdout.log|ROOT_CAUSE_COUNT=1 +stdout.log|DETAIL=phase=preflight line=13 rc=1 +RUN_FILE=stderr.log|present=true|bytes=0 +RUN_FILE=preflight.json|present=false|bytes=0 +RUN_FILE=result.json|present=true|bytes=592 +result.json|{ +result.json| "run_id": "20260923T120316Z-986653-preflight", +result.json| "task_id": "skladchik-reports-monitor-controlled-reauth-v1", +result.json| "phase": "preflight", +result.json| "mutation_scope": "controlled_apply", +result.json| "production_change": true, +result.json| "started_at": "2026-09-23T12:03:16Z", +result.json| "finished_at": "2026-09-23T12:03:16Z", +result.json| "status": "FAILED", +result.json| "rc": 1, +result.json| "stdout": "/var/lib/homelab-admin/runs/skladchik-reports-monitor-controlled-reauth-v1/20260923T120316Z-986653-preflight/stdout.log", +result.json| "stderr": "/var/lib/homelab-admin/runs/skladchik-reports-monitor-controlled-reauth-v1/20260923T120316Z-986653-preflight/stderr.log" +result.json|} +RUN_FILE=task.json|present=true|bytes=13579 +task.json|{ +task.json| "schema_version": 2, +task.json| "task_id": "skladchik-reports-monitor-controlled-reauth-v1", +task.json| "target_host": "pve01.gram1.ru", +task.json| "mutation_scope": "controlled_apply", +task.json| "production_change": true, +task.json| "failure_mode": "fail_fast", +task.json| "guards": { +task.json| "desired_state_clean": true, +task.json| "failed_units_max": 4, +task.json| "active_backup_restore_max": 0 +task.json| }, +task.json| "reads": [ +task.json| "[SENSITIVE_PATH] +task.json| "/var/lib/skladchik-reports-monitor-backup/health.txt", +task.json| "/var/lib/skladchik-reports-monitor-backup/restore-check-health.txt", +task.json| "systemd:skladchik-reports-monitor-supervisor.service", +task.json| "systemd:skladchik-reports-monitor-supervisor.timer", +task.json| "ssh:debian@[PRIVATE_IP]:/usr/local/sbin/skladchik-reports-monitor-cookie-update-edge", +task.json| "ssh:debian@[PRIVATE_IP]:/etc/skladchik-reports-monitor/secrets.env", +task.json| "ssh:debian@[PRIVATE_IP]:/var/lib/skladchik-reports-monitor/cookies.jar", +task.json| "ssh:debian@[PRIVATE_IP]:/var/lib/skladchik-reports-monitor/health.txt", +task.json| "/usr/local/sbin/skladchik-reports-monitor-cookie-update", +task.json| "/usr/local/sbin/skladchik-reports-monitor-backup", +task.json| "/usr/local/sbin/skladchik-reports-monitor-monthly-selftest-watch", +task.json| "/usr/local/sbin/skladchik-reports-monitor-status", +task.json| "/usr/local/sbin/skladchik-reports-monitor-supervisor", +task.json| "/srv/homelab-desired-state/sbin/skladchik-reports-monitor-backup", +task.json| "/var/lib/skladchik-reports-monitor-supervisor" +task.json| ], +task.json| "writes": [ +task.json| "/var/lib/homelab-admin/seals/skladchik-reports-monitor-controlled-reauth-v1.json", +task.json| "systemd:skladchik-reports-monitor-supervisor.service", +task.json| "ssh:debian@[PRIVATE_IP]:/etc/skladchik-reports-monitor/secrets.env", +task.json| "ssh:debian@[PRIVATE_IP]:/var/lib/skladchik-reports-monitor/cookies.jar", +task.json| "ssh:debian@[PRIVATE_IP]:/var/lib/skladchik-reports-monitor/health.txt", +task.json| "ssh:debian@[PRIVATE_IP]:/var/lib/homelab-admin/rollback/skladchik-reports-monitor-controlled-reauth-v1", +task.json| "$HOMELAB_RUN_DIR/preflight.json", +task.json| "$HOMELAB_RUN_DIR/prepare.json", +task.json| "$HOMELAB_RUN_DIR/dry-run.json", +task.json| "$HOMELAB_RUN_DIR/apply.json", +task.json| "$HOMELAB_RUN_DIR/verify.json", +task.json| "$HOMELAB_RUN_DIR/rollback.json", +task.json| "$HOMELAB_RUN_DIR/seal.json", +task.json| "/usr/local/sbin/skladchik-reports-monitor-cookie-update", +task.json| "/usr/local/sbin/skladchik-reports-monitor-backup", +task.json| "/usr/local/sbin/skladchik-reports-monitor-monthly-selftest-watch", +task.json| "/usr/local/sbin/skladchik-reports-monitor-status", +task.json| "/usr/local/sbin/skladchik-reports-monitor-supervisor", +task.json| "/srv/homelab-desired-state/sbin/skladchik-reports-monitor-cookie-update", +task.json| "/srv/homelab-desired-state/sbin/skladchik-reports-monitor-backup", +task.json| "/srv/homelab-desired-state/sbin/skladchik-reports-monitor-monthly-selftest-watch", +task.json| "/srv/homelab-desired-state/sbin/skladchik-reports-monitor-status", +task.json| "/srv/homelab-desired-state/sbin/skladchik-reports-monitor-supervisor", +task.json| "/mnt/staging/homelab-admin-rollback/skladchik-reports-monitor-controlled-reauth-v1", +task.json| "ssh:debian@[PRIVATE_IP]:/usr/local/sbin/skladchik-reports-monitor-cookie-update-edge" +task.json| ], +task.json| "mutates": [ +task.json| "systemd:skladchik-reports-monitor-supervisor.service", +task.json| "ssh:debian@[PRIVATE_IP]:/etc/skladchik-reports-monitor/secrets.env", +task.json| "ssh:debian@[PRIVATE_IP]:/var/lib/skladchik-reports-monitor/cookies.jar", +task.json| "ssh:debian@[PRIVATE_IP]:systemd:skladchik-reports-monitor.service", +task.json| "ssh:debian@[PRIVATE_IP]:systemd:skladchik-reports-monitor.timer", +task.json| "/usr/local/sbin/skladchik-reports-monitor-cookie-update", +task.json| "/usr/local/sbin/skladchik-reports-monitor-backup", +task.json| "/usr/local/sbin/skladchik-reports-monitor-monthly-selftest-watch", +task.json| "/usr/local/sbin/skladchik-reports-monitor-status", +task.json| "/usr/local/sbin/skladchik-reports-monitor-supervisor", +task.json| "git:/srv/homelab-desired-state", +task.json| "ssh:debian@[PRIVATE_IP]:/usr/local/sbin/skladchik-reports-monitor-cookie-update-edge" +task.json| ], +task.json| "expected_outputs": [ +task.json| { +task.json| "path": "$HOMELAB_RUN_DIR/preflight.json", +task.json| "producer": "phases/preflight.sh", +task.json| "verification": "strict host-key access, the pinned live predecessor edge script, the versioned target edge script, AUTH_REQUIRED and bounded monitor quiescence are confirmed; evidence records both SHAs" +task.json| }, +task.json| { +task.json| "path": "$HOMELAB_RUN_DIR/prepare.json", +task.json| "producer": "phases/prepare.sh", +task.json| "verification": "root-only local and edge rollback snapshots have self-contained SHA256 manifests" +task.json| }, +task.json| { +task.json| "path": "$HOMELAB_RUN_DIR/dry-run.json", +task.json| "producer": "phases/dry-run.sh", +task.json| "verification": "rollback manifests validate, the pre-apply live edge script remains the pinned predecessor, the versioned deployment asset matches the pinned target SHA256, and evidence records both SHAs" +task.json| }, +task.json| { +task.json| "path": "$HOMELAB_RUN_DIR/apply.json", +task.json| "producer": "phases/apply.sh", +task.json| "verification": "apply replaces the pinned predecessor edge script with the pinned target script, quiesces the timer, performs one controlled reauthentication and restores or enables the timer" +task.json| }, +task.json| { +task.json| "path": "$HOMELAB_RUN_DIR/verify.json", +task.json| "producer": "phases/verify.sh", +task.json| "verification": "edge monitor, counts, timer, supervisor, backup and restore are OK" +task.json| }, +task.json| { +task.json| "path": "$HOMELAB_RUN_DIR/rollback.json", +task.json| "producer": "phases/rollback.sh", +task.json| "verification": "previous auth artifacts and timer state are restored" +task.json| }, +task.json| { +task.json| "path": "$HOMELAB_RUN_DIR/seal.json", +task.json| "producer": "phases/seal.sh", +task.json| "verification": "standard seal exists and rollback snapshot is removed" +task.json| } +task.json| ], +task.json| "runtime_matrix": [ +task.json| { +task.json| "host": "pve01.gram1.ru", +task.json| "runtime": "bash,jq,git,systemd,ssh", +task.json| "compile": "bash -n; jq -e" +task.json| }, +task.json| { +task.json| "host": "edge", +task.json| "runtime": "bash,python3,systemd,sha256sum", +task.json| "compile": "bash -n" +task.json| } +task.json| ], +task.json| "rollback": { +task.json| "command": "HOMELAB_CONTROLLED_APPLY_CONFIRM=skladchik-reports-monitor-controlled-reauth-v1 /opt/homelab-admin/bin/homelab-admin run TASK_DIR rollback", +task.json| "verification": "self-contained local and edge manifests validate; prior auth artifacts, scripts, desired-state HEAD, supervisor state and timer state are restorable" +task.json| }, +task.json| "no_change_assertions": [ +task.json| "vm160_backup_job", +task.json| "cluster_backup_jobs", +task.json| "unrelated_guests", +task.json| "network", +task.json| "dns", +task.json| "known_hosts", +task.json| "mail_cloud", +task.json| "gitea", +task.json| "backup_and_restore_artifacts" +task.json| ], +task.json| "source_of_truth": { +task.json| "change_id": "CR-2026-0012", +task.json| "incident": "Skladchik monitor AUTH_REQUIRED with missing complaint counts", +task.json| "edge_script_sha256": "10fe854274dca1a053cfd8c0155340bd689bf3f6555c70dbc14b890cbafb93fa", +task.json| "edge_script_path": "/usr/local/sbin/skladchik-reports-monitor-cookie-update-edge", +task.json| "unsafe_legacy_wrapper_path": "/usr/local/sbin/skladchik-reports-monitor-cookie-update", +task.json| "unsafe_legacy_wrapper_reason": "legacy host-key auto-acceptance", +task.json| "strict_hostkey_required": true, +task.json| "hidden_interactive_prompt_required": true, +task.json| "cookie_content_must_not_enter_git_or_logs": true, +task.json| "backup_status_before": "OK", +task.json| "restore_status_before": "OK", +task.json| "regression_lock": "REG-024-SKLADCHIK-REAUTH-STRICT-HOSTKEY", +task.json| "active_pve01_script_count": 5, +task.json| "active_pve01_original_sha256": { +task.json| "skladchik-reports-monitor-cookie-update": "a3006723d902fb7735c593003a6c6755fa3d69683126ec5f53165131f754fd7a", +task.json| "skladchik-reports-monitor-backup": "ad0652d1e154fca94a20bdf831b40b831fd0d862eb104ec3377f486e96270d0a", +task.json| "skladchik-reports-monitor-monthly-selftest-watch": "dd2c7e26aaa6835fa4ac4365eade338eb7b19f4bfe7528179c59cf8039454180", +task.json| "skladchik-reports-monitor-status": "f94ec75155f415b35517dade0bfba00926eba6088e0c2b5c96ae79daa5dc368f", +task.json| "skladchik-reports-monitor-supervisor": "69494a4c32fa632844cdb54aabbe302a14d478924efcb719c6cd3d594d102369" +task.json| }, +task.json| "active_pve01_deployed_sha256": { +task.json| "skladchik-reports-monitor-cookie-update": "ea2d89ce67893a6d89b29610e4d46e207932ecc157173454585f67890f58810c", +task.json| "skladchik-reports-monitor-backup": "1912f8f0559637563972bb8ac984fb0036694092056ae53968c02cc3b67855f3", +task.json| "skladchik-reports-monitor-monthly-selftest-watch": "48c6ca01d2430d14c6af797c079640815ba1352a98466d4014ecbb80a27eaada", +task.json| "skladchik-reports-monitor-status": "f902a61109efbac5918360f2235c2523bd10704ac484a4e8591e8a1c490faf5a", +task.json| "skladchik-reports-monitor-supervisor": "35c68ae0929ff3795be2bde40de334fc8e26413a3d8a620a137be90d1f2c164e" +task.json| }, +task.json| "edge_mutation_script_versioned": true, +task.json| "desired_state_script_count_after_apply": 5, +task.json| "all_active_skladchik_ssh_requires_strict_known_hosts": true, +task.json| "unsafe_active_hostkey_markers_after_verify": 0, +task.json| "pve01_assets_checked_by_repository_tests": true, +task.json| "regression_locks": [ +task.json| "REG-024-SKLADCHIK-REAUTH-STRICT-HOSTKEY", +task.json| "REG-025-EXACT-SHA256-LENGTH-VALIDATION", +task.json| "REG-026-ACTIVE-SKLADCHIK-STRICT-HOSTKEY-COVERAGE", +task.json| "REG-027-REMOTE-MUTATION-SCRIPT-MUST-BE-VERSIONED", +task.json| "REG-028-FORBIDDEN-TOKEN-IN-NEGATIVE-ASSERTION", +task.json| "REG-029-LEGACY-WRAPPER-NOT-COPIED-VERBATIM", +task.json| "REG-030-BINARY-ARTIFACT-TEXT-SCAN", +task.json| "REG-031-SEMANTIC-TEST-LOOP-EXPANSION-MISMATCH", +task.json| "REG-032-OVERBROAD-INLINE-PYTHON-TEST", +task.json| "REG-033-INSTALLED-SOURCE-MUST-BE-DECLARED", +task.json| "REG-034-ROLLBACK-STAGE-PARENT-MODE", +RUN_EVIDENCE_END +CURRENT_FACTS_BEGIN +EDGE_SCRIPT_LOCAL_EXISTS=true +EDGE_SCRIPT_LOCAL_META=mode=700|uid=0|gid=0|size=5247 +EDGE_SCRIPT_LOCAL_SHA256=10fe854274dca1a053cfd8c0155340bd689bf3f6555c70dbc14b890cbafb93fa +LOCAL_HEALTH_EXISTS=false +SECRETS_ENV_EXISTS=false +MONITOR_PY_EXISTS=false +UNIT=skladchik-reports-monitor.service|active=inactive|enabled=not-found +UNIT=skladchik-reports-monitor.timer|active=inactive|enabled=not-found +UNIT=skladchik-reports-monitor-supervisor.service|active=failed|enabled=static +UNIT=skladchik-reports-monitor-supervisor.timer|active=active|enabled=enabled +CURRENT_FACTS_END +RCA_CAPTURE_COMPLETE=true +CHANGES_MADE_BY_943=false +ROLLBACK_STARTED=false +ROLLBACK_RESTORED=null +HOMELAB_RESULT_CONTRACT={"version":1,"command_id":"SUPPORT-260923-SKLADCHIK-PREFLIGHT-LINE13-RCA-943R1","status":"OK","changes_made":false,"rollback_started":false,"rollback_restored":null} +SKLADCHIK943_END=true + +OUTPUT_END +CHAT_OUTPUT_END diff --git a/runtime/latest.json b/runtime/latest.json index fcb69caa..f54056ab 100644 --- a/runtime/latest.json +++ b/runtime/latest.json @@ -1,38 +1,38 @@ { "schema_version": 1, "channel": "homelab-runtime", - "command_id": "SUPPORT-260923-SKLADCHIK-COOKIE-REFRESH-REISSUE-942R1", - "status": "FAIL", - "rc": 1, + "command_id": "SUPPORT-260923-SKLADCHIK-PREFLIGHT-LINE13-RCA-943R1", + "status": "OK", + "rc": 0, "host": "pve01", - "mode": "change", - "component": "skladchik-cookie-refresh-reissue", - "started_at_utc": "2026-09-23T12:03:03Z", - "finished_at_utc": "2026-09-23T12:03:19Z", + "mode": "read-only", + "component": "skladchik-preflight-line13-rca", + "started_at_utc": "2026-09-23T12:08:41Z", + "finished_at_utc": "2026-09-23T12:08:41Z", "reference_register_checked": true, "reference_sha256": "5a3d8e5154c41cb582a4e0aca68090be1f0138918bf82131a948df326f9d8d66", "error_register_checked": true, "error_register_sha256": "3b09a553ec0f527ed3afeed4753f52a74ee3036045bbfb3c685e9f8af4ba7ba0", - "command_sha256": "27da7524dbfc342b7925b1034305a552a30bcb635ca4e09e5ee7c3bfdb3e51b6", + "command_sha256": "dd3a88084befa0f31cc5f49ef676cb01eec2dfd507c1e1bfd993fba637e753d7", "duplicate_failed_command_blocked": false, "block_reason": null, "execution_started": true, - "change_declared": true, + "change_declared": false, "result_contract_valid": true, - "result_contract_status": "FAIL", + "result_contract_status": null, "result_contract_error": null, - "command_rc": 1, - "changes_made": true, + "command_rc": 0, + "changes_made": false, "rollback_started": false, "rollback_restored": null, - "mutation_outcome": "MUTATED", + "mutation_outcome": "NO_MUTATION", "sanitized": true, "secrets_included": false, "private_addresses_included": false, "raw_evidence_retained_locally": true, - "raw_evidence_sha256": "edad064e291af1b094dd2a4d47ae4372a5fa3ddfdb4518232751e42562c271f5", - "sanitized_output_sha256": "a51b6642d8cde7b817b5bfd3ee7d5b5a3b22459a9dee1f3d368a3ef292aebb3e", + "raw_evidence_sha256": "2c42ff80da827c41e303d1369683a91c76fe254e8a5cc304eeb6038334a8dafa", + "sanitized_output_sha256": "17ace4d0e724d6b41d9460dad37dfe85e5b452f724ea763d0ba457d6ad255c77", "output_truncated_in_json": false, "full_sanitized_output_url": "https://git.gram1.ru/.well-known/homelab-runtime/latest.txt", - "output": "SKLADCHIK942_BEGIN=true\nCOMMAND_ID=SUPPORT-260923-SKLADCHIK-COOKIE-REFRESH-REISSUE-942R1\nMODE=change\nCOMPONENT=skladchik-cookie-refresh-reissue\nPREDECESSOR_929=PASS_SKLADCHIK_929_TRANSPORT_RECONCILE_SAFE_TO_REISSUE\nPREDECESSOR_931=LOCAL_TRANSPORT_RC126_NO_RUNNER_RESULT\nPREDECESSOR_932=PASS_SKLADCHIK_931_LOCAL_CARRIER_FORENSICS\nPREDECESSOR_933=HOLD_SKLADCHIK_933_COOKIE_STDIN_INVALID_NO_MUTATION\nPREDECESSOR_934=HOLD_SKLADCHIK_934_COOKIE_STDIN_INVALID_NO_MUTATION\n931_RCA=UPLOAD_SUCCEEDED;DIRECT_EXEC_FROM_RUN_RETURNED_126;STDERR_63_BYTES_MATCHES_BASH_PERMISSION_DENIED;FIX_RUN_OUTER_VIA_USR_BIN_BASH\n933_RCA=RUNNER_REACHED;COOKIE_CLIPBOARD_STDIN_INVALID;NO_MUTATION\n934_RCA=RUNNER_REACHED;READ_SUCCEEDED_BUT_COOKIE_VALIDATION_FAILED;NO_MUTATION;COMMON_FAILURE_IS_WINDOWS_INPUT_TRANSPORT\n935_TRANSPORT_PROOF=MASKED_GUI_TEXTBOX_WITH_PASTE;DIRECT_WRITE_TO_PLINK_PROCESS_STANDARDINPUT;COOKIE_INPUT_LENGTH_325_REACHED_RUNNER\nPREDECESSOR_935=OFFICIAL_REFRESH_REACHED_INITIAL_STATE_CHECK;FAILED_BEFORE_COOKIE_PROMPT\nPREDECESSOR_939=ONLY_INITIAL_CHECK_LINE_159_FAILED\nPREDECESSOR_940=LOCAL_BEHIND_FAST_FORWARD_TO_ORIGIN\nPREDECESSOR_941=PASS_HOMELAB_OPS_FASTFORWARD;INITIAL_STATE_CHECK_ALL_PASS\n942_PURPOSE=REISSUE_OFFICIAL_COOKIE_REFRESH_AFTER_LINE_159_REMEDIATION\nMUTATION_BOUNDARY=OFFICIAL_SKLADCHIK_COOKIE_REFRESH_ONLY;TRANSIENT_ROOT_ONLY_COOKIE_STAGING;NO_OTHER_SERVICES;NO_VM_CONFIG;NO_CLOUD_CONFIG\nCOOKIE_INPUT_LENGTH=325\nINITIAL_STATE_GIT_GATE=PASS\nRUNNER_FIRST_COOKIE_INPUT=true\nOFFICIAL_RUNBOOK_VERIFIED=true\nCOOKIE_SOURCE=WINDOWS_CHROME_MASKED_GUI_DIRECT_PROCESS_STANDARDINPUT\nCOOKIE_SECRET_OUTPUT=NONE\nSCOPE=SKLADCHIK_COOKIE_REFRESH_VALIDATE_PREFLIGHT_PREPARE_DRY_RUN_HIDDEN_COOKIE_APPLY_VERIFY_SEAL\nROLLBACK=AUTOMATIC_VERSIONED_RECOVERY_WORKFLOW_RESTORES_PREVIOUS_SECRET_COOKIE_JAR_SCRIPTS_DESIRED_STATE_TIMER_AND_SUPERVISOR_STATE_ON_FAILURE_BEFORE_SEAL\nVERIFY=FRESH_COOKIE_FORCED_STALE_JAR_BYPASSED_SECOND_JAR_ONLY_PROBE_MONITOR_OK_NUMERIC_COUNTS_TARGET_EDGE_SHA_TIMERS_SUPERVISOR_SEAL_STAGE_REMOVAL_VM160_UNCHANGED_NO_CLOUD_COMMANDS\nLINT_STATUS=OK\nTASK_ID=skladchik-reports-monitor-controlled-reauth-v1\nPOLICY_ID=homelab-admin-guardrails-v1.2\nRULES_CHECKED=14\nSOURCE_FILES_CHECKED=19\nTASK_VALID=YES\nTASK_ID=skladchik-reports-monitor-controlled-reauth-v1\nTASK_SCOPE=controlled_apply\nPRODUCTION_CHANGE=YES\nLINT_STATUS=OK\nTASK_ID=skladchik-reports-monitor-controlled-reauth-v1\nPOLICY_ID=homelab-admin-guardrails-v1.2\nRULES_CHECKED=14\nSOURCE_FILES_CHECKED=19\nROOT_CAUSE=UNEXPECTED_PHASE_ERROR\nROOT_CAUSE_COUNT=1\nDETAIL=phase=preflight line=13 rc=1\nRUN_DIR=/var/lib/homelab-admin/runs/skladchik-reports-monitor-controlled-reauth-v1/20260923T120316Z-986653-preflight\nRUN_RC=1\n[1;35m===== СКОПИРОВАТЬ МНЕ В ЧАТ =====\nCOOKIE_REFRESH_RESULT=FAILED\nFAILED_STAGE=PREFLIGHT\nFAILED_RC=1\nROLLBACK_ATTEMPTED=NO\nROLLBACK_RC=NOT_RUN\nSEALED=1\nBACKUP_WAIT_EVENTS=0\nACTIVE_BACKUP_RESTORE_NOW=0\nAPPLY_RUN=unknown\nCOOKIE_VALUES_PRINTED=NO\nVM160_CHANGED=NO\nCLOUD_COMMANDS_EXECUTED=NO\nNEXT_PHASE=PASTE_THIS_FRAGMENT_ONLY\n===== КОНЕЦ ФРАГМЕНТА =====[0m\nPTY_COOKIE_FEED_SENT=false\nPTY_COOKIE_PROMPT_SEEN=false\nOFFICIAL_REFRESH_RC=1\nPTY_COOKIE_FEED_SENT=false\nPTY_COOKIE_PROMPT_SEEN=false\nCOOKIE_REFRESH_RESULT_SUCCESS=false\nFRESH_COOKIE_FORCED=false\nSTALE_COOKIE_JAR_BYPASSED=false\nPOST_SUPERVISOR_STATUS=FAIL\nPOST_MONITOR_STATUS=ERROR\nPOST_MONITOR_REPORTS=missing\nPOST_SESSION_KEEPALIVE=persistent_cookie_jar\nPOST_COOKIE_EXPIRY_STATUS=WARN\nPOST_COOKIE_JAR_PRESENT=false\nPOST_COOKIE_JAR_BYTES=0\nPOST_NETBIRD_ROUTE_WT0=false\nPOST_ROUTE_PROBLEM_REPORTED=true\nFAILURE=OFFICIAL_REFRESH_RC_1\nCOOKIE_REFRESH_COMPLETE=false\nDECISION=HOLD_SKLADCHIK_COOKIE_REFRESH_RCA\nNEXT_GATE=SKLADCHIK_COOKIE_REFRESH_RCA\nTASK_RESULT=HOLD_SKLADCHIK_COOKIE_REFRESH\nCHANGES_MADE_BY_942=true\nROLLBACK_STARTED=false\nROLLBACK_RESTORED=null\nHOMELAB_RESULT_CONTRACT={\"version\":1,\"command_id\":\"SUPPORT-260923-SKLADCHIK-COOKIE-REFRESH-REISSUE-942R1\",\"status\":\"FAIL\",\"changes_made\":true,\"rollback_started\":false,\"rollback_restored\":null}\nSKLADCHIK942_END=true\n" + "output": "SKLADCHIK943_BEGIN=true\nCOMMAND_ID=SUPPORT-260923-SKLADCHIK-PREFLIGHT-LINE13-RCA-943R1\nMODE=read-only\nCOMPONENT=skladchik-preflight-line13-rca\nPREDECESSOR_942=FAILED_STAGE_PREFLIGHT;DETAIL_phase_preflight_line_13_rc_1\nMUTATION_BOUNDARY=NONE_READ_ONLY\nPHASE_PREFLIGHT_SOURCE_BEGIN\nPHASE_PREFLIGHT_SOURCE_PRESENT=true\nPHASE_PREFLIGHT_SOURCE_SHA256=f3e533ebd72ba2343a78797a8087c529c9fcfa2b3cabc45b3c0bd477356a8cc9\nPHASE_PREFLIGHT_SOURCE_BYTES=3076\nPHASE_PREFLIGHT_SOURCE_LINES=29\n0001:#!/usr/bin/env bash\n0002:set -Eeuo pipefail\n0003:on_err(){ local rc=$?; trap - ERR; printf 'ROOT_CAUSE=UNEXPECTED_PHASE_ERROR\\nROOT_CAUSE_COUNT=1\\nDETAIL=phase=preflight line=%s rc=%s\\n' \"${1:-0}\" \"$rc\"; exit \"$rc\"; }\n0004:trap 'on_err \"$LINENO\"' ERR\n0005:export SYSTEMD_PAGER=cat PAGER=cat SYSTEMD_LESS=FRXMK\n0006:TASK_ID=skladchik-reports-monitor-controlled-reauth-v1\n0007:EDGE=debian@[PRIVATE_IP]\n0008:CURRENT_EDGE_SHA=b13d846cf089d450e3d5ac9e85ad1e232c198930e466bb8b82f6983a4d48ce0e\n0009:TARGET_EDGE_SHA=10fe854274dca1a053cfd8c0155340bd689bf3f6555c70dbc14b890cbafb93fa\n0010:SSH=(-o BatchMode=yes -o ConnectTimeout=12 -o StrictHostKeyChecking=yes -o UserKnownHostsFile=[SENSITIVE_PATH]\n0011:test \"$(hostname -f)\" = pve01.gram1.ru\n0012:test -z \"$(git -C /srv/homelab-desired-state status --porcelain)\"\n0013:test \"$(sha256sum /usr/local/sbin/skladchik-reports-monitor-cookie-update | awk '{print $1}')\" = a3006723d902fb7735c593003a6c6755fa3d69683126ec5f53165131f754fd7a\n0014:test \"$(sha256sum /usr/local/sbin/skladchik-reports-monitor-backup | awk '{print $1}')\" = ad0652d1e154fca94a20bdf831b40b831fd0d862eb104ec3377f486e96270d0a\n0015:test \"$(sha256sum /usr/local/sbin/skladchik-reports-monitor-monthly-selftest-watch | awk '{print $1}')\" = dd2c7e26aaa6835fa4ac4365eade338eb7b19f4bfe7528179c59cf8039454180\n0016:test \"$(sha256sum /usr/local/sbin/skladchik-reports-monitor-status | awk '{print $1}')\" = f94ec75155f415b35517dade0bfba00926eba6088e0c2b5c96ae79daa5dc368f\n0017:test \"$(sha256sum /usr/local/sbin/skladchik-reports-monitor-supervisor | awk '{print $1}')\" = 69494a4c32fa632844cdb54aabbe302a14d478924efcb719c6cd3d594d102369\n0018:test \"$(sha256sum /srv/homelab-desired-state/sbin/skladchik-reports-monitor-backup | awk '{print $1}')\" = ad0652d1e154fca94a20bdf831b40b831fd0d862eb104ec3377f486e96270d0a\n0019:test \"$(sha256sum \"$HOMELAB_TASK_DIR/assets/edge/skladchik-reports-monitor-cookie-update-edge\" | awk '{print $1}')\" = \"$TARGET_EDGE_SHA\"\n0020:RESULT=\"$(ssh \"${SSH[@]}\" \"$EDGE\" sudo -n /bin/bash -s -- \"$CURRENT_EDGE_SHA\" < \"$HOMELAB_TASK_DIR/assets/edge-preflight.sh\")\"\n0021:grep -Fq 'EDGE_PREFLIGHT_STATUS=OK' <<<\"$RESULT\"\n0022:test \"$(systemctl is-enabled skladchik-reports-monitor-supervisor.timer)\" = enabled\n0023:test \"$(systemctl is-active skladchik-reports-monitor-supervisor.timer)\" = active\n0024:test \"$(systemctl show skladchik-reports-monitor-supervisor.service -p Result --value)\" = exit-code\n0025:grep -q '^STATUS=OK$' /var/lib/skladchik-reports-monitor-backup/health.txt\n0026:grep -q '^STATUS=OK$' /var/lib/skladchik-reports-monitor-backup/restore-check-health.txt\n0027:jq -n --arg status OK --arg current_edge_script_sha \"$CURRENT_EDGE_SHA\" --arg target_edge_script_sha \"$TARGET_EDGE_SHA\" '{status:$status,root_cause_count:0,production_changed:false,edge_monitor_status:\"AUTH_REQUIRED\",current_edge_script_sha256:$current_edge_script_sha,target_edge_script_sha256:$target_edge_script_sha,strict_hostkey:true,active_skladchik_sources_pinned:true}' > \"$HOMELAB_RUN_DIR/preflight.json\"\n0028:printf '%s\\n' \"$RESULT\"\n0029:printf 'PREFLIGHT_STATUS=OK\\nACTIVE_SKLADCHIK_SOURCE_HASHES=PINNED\\nEDGE_MUTATION_SCRIPT=VERSIONED\\nROOT_CAUSE_COUNT=0\\nPRODUCTION_CHANGED=NO\\nREAUTH_EXECUTED=NO\\n'\nPHASE_PREFLIGHT_SOURCE_END\nEDGE_PREFLIGHT_SOURCE_BEGIN\nEDGE_PREFLIGHT_SOURCE_PRESENT=true\nEDGE_PREFLIGHT_SOURCE_SHA256=77ebb2893962550f97d80bdfd2eb414f666578a272b6c35a3f4ca669df117878\nEDGE_PREFLIGHT_SOURCE_BYTES=1351\nEDGE_PREFLIGHT_SOURCE_LINES=29\n0001:#!/usr/bin/env bash\n0002:set -Eeuo pipefail\n0003:EXPECTED_SHA=\"${1:?expected script sha required}\"\n0004:SCRIPT=/usr/local/sbin/skladchik-reports-monitor-cookie-update-edge\n0005:HEALTH=/var/lib/skladchik-reports-monitor/health.txt\n0006:test -x \"$SCRIPT\"\n0007:test \"$(sha256sum \"$SCRIPT\" | awk '{print $1}')\" = \"$EXPECTED_SHA\"\n0008:test -s /etc/skladchik-reports-monitor/secrets.env\n0009:test -s /opt/skladchik-reports-monitor/monitor.py\n0010:test -s \"$HEALTH\"\n0011:STATUS=\"$(awk -F= '$1==\"STATUS\"{print $2;exit}' \"$HEALTH\")\"\n0012:test \"$STATUS\" = AUTH_REQUIRED\n0013:test \"$(systemctl is-enabled skladchik-reports-monitor.timer)\" = enabled\n0014:test \"$(systemctl is-active skladchik-reports-monitor.timer)\" = active\n0015:WAIT_SECONDS=90\n0016:DEADLINE=$((SECONDS+WAIT_SECONDS))\n0017:monitor_busy() {\n0018: systemctl is-active --quiet skladchik-reports-monitor.service ||\n0019: pgrep -f '/opt/skladchik-reports-monitor/monitor.py' >/dev/null ||\n0020: pgrep -f 'skladchik-reports-monitor-cookie-update-edge' >/dev/null\n0021:}\n0022:while monitor_busy; do\n0023: if (( SECONDS >= DEADLINE )); then\n0024: printf 'EDGE_PREFLIGHT_STATUS=BUSY\\nFAILED_PHASE=WAIT_MONITOR_QUIESCENCE\\nWAIT_SECONDS=%s\\n' \"$WAIT_SECONDS\"\n0025: exit 75\n0026: fi\n0027: sleep 2\n0028:done\n0029:printf 'EDGE_PREFLIGHT_STATUS=OK\\nEDGE_MONITOR_STATUS=%s\\nEDGE_SCRIPT_SHA256=%s\\nEDGE_TIMER_ENABLED=YES\\nEDGE_TIMER_ACTIVE=YES\\nMONITOR_QUIESCENCE=OK\\nWAIT_LIMIT_SECONDS=%s\\n' \"$STATUS\" \"$EXPECTED_SHA\" \"$WAIT_SECONDS\"\nEDGE_PREFLIGHT_SOURCE_END\nFAILED_LINE_CONTEXT_BEGIN\nPHASE_LINE_0009=TARGET_EDGE_SHA=10fe854274dca1a053cfd8c0155340bd689bf3f6555c70dbc14b890cbafb93fa\nPHASE_LINE_0010=SSH=(-o BatchMode=yes -o ConnectTimeout=12 -o StrictHostKeyChecking=yes -o UserKnownHostsFile=[SENSITIVE_PATH]\nPHASE_LINE_0011=test \"$(hostname -f)\" = pve01.gram1.ru\nPHASE_LINE_0012=test -z \"$(git -C /srv/homelab-desired-state status --porcelain)\"\nPHASE_LINE_0013=test \"$(sha256sum /usr/local/sbin/skladchik-reports-monitor-cookie-update | awk '{print $1}')\" = a3006723d902fb7735c593003a6c6755fa3d69683126ec5f53165131f754fd7a\nPHASE_LINE_0014=test \"$(sha256sum /usr/local/sbin/skladchik-reports-monitor-backup | awk '{print $1}')\" = ad0652d1e154fca94a20bdf831b40b831fd0d862eb104ec3377f486e96270d0a\nPHASE_LINE_0015=test \"$(sha256sum /usr/local/sbin/skladchik-reports-monitor-monthly-selftest-watch | awk '{print $1}')\" = dd2c7e26aaa6835fa4ac4365eade338eb7b19f4bfe7528179c59cf8039454180\nPHASE_LINE_0016=test \"$(sha256sum /usr/local/sbin/skladchik-reports-monitor-status | awk '{print $1}')\" = f94ec75155f415b35517dade0bfba00926eba6088e0c2b5c96ae79daa5dc368f\nPHASE_LINE_0017=test \"$(sha256sum /usr/local/sbin/skladchik-reports-monitor-supervisor | awk '{print $1}')\" = 69494a4c32fa632844cdb54aabbe302a14d478924efcb719c6cd3d594d102369\nEDGE_LINE_0009=test -s /opt/skladchik-reports-monitor/monitor.py\nEDGE_LINE_0010=test -s \"$HEALTH\"\nEDGE_LINE_0011=STATUS=\"$(awk -F= '$1==\"STATUS\"{print $2;exit}' \"$HEALTH\")\"\nEDGE_LINE_0012=test \"$STATUS\" = AUTH_REQUIRED\nEDGE_LINE_0013=test \"$(systemctl is-enabled skladchik-reports-monitor.timer)\" = enabled\nEDGE_LINE_0014=test \"$(systemctl is-active skladchik-reports-monitor.timer)\" = active\nEDGE_LINE_0015=WAIT_SECONDS=90\nEDGE_LINE_0016=DEADLINE=$((SECONDS+WAIT_SECONDS))\nEDGE_LINE_0017=monitor_busy() {\nFAILED_LINE_CONTEXT_END\nRUN_EVIDENCE_BEGIN\nRUN_FILE=stdout.log|present=true|bytes=89\nstdout.log|ROOT_CAUSE=UNEXPECTED_PHASE_ERROR\nstdout.log|ROOT_CAUSE_COUNT=1\nstdout.log|DETAIL=phase=preflight line=13 rc=1\nRUN_FILE=stderr.log|present=true|bytes=0\nRUN_FILE=preflight.json|present=false|bytes=0\nRUN_FILE=result.json|present=true|bytes=592\nresult.json|{\nresult.json| \"run_id\": \"20260923T120316Z-986653-preflight\",\nresult.json| \"task_id\": \"skladchik-reports-monitor-controlled-reauth-v1\",\nresult.json| \"phase\": \"preflight\",\nresult.json| \"mutation_scope\": \"controlled_apply\",\nresult.json| \"production_change\": true,\nresult.json| \"started_at\": \"2026-09-23T12:03:16Z\",\nresult.json| \"finished_at\": \"2026-09-23T12:03:16Z\",\nresult.json| \"status\": \"FAILED\",\nresult.json| \"rc\": 1,\nresult.json| \"stdout\": \"/var/lib/homelab-admin/runs/skladchik-reports-monitor-controlled-reauth-v1/20260923T120316Z-986653-preflight/stdout.log\",\nresult.json| \"stderr\": \"/var/lib/homelab-admin/runs/skladchik-reports-monitor-controlled-reauth-v1/20260923T120316Z-986653-preflight/stderr.log\"\nresult.json|}\nRUN_FILE=task.json|present=true|bytes=13579\ntask.json|{\ntask.json| \"schema_version\": 2,\ntask.json| \"task_id\": \"skladchik-reports-monitor-controlled-reauth-v1\",\ntask.json| \"target_host\": \"pve01.gram1.ru\",\ntask.json| \"mutation_scope\": \"controlled_apply\",\ntask.json| \"production_change\": true,\ntask.json| \"failure_mode\": \"fail_fast\",\ntask.json| \"guards\": {\ntask.json| \"desired_state_clean\": true,\ntask.json| \"failed_units_max\": 4,\ntask.json| \"active_backup_restore_max\": 0\ntask.json| },\ntask.json| \"reads\": [\ntask.json| \"[SENSITIVE_PATH]\ntask.json| \"/var/lib/skladchik-reports-monitor-backup/health.txt\",\ntask.json| \"/var/lib/skladchik-reports-monitor-backup/restore-check-health.txt\",\ntask.json| \"systemd:skladchik-reports-monitor-supervisor.service\",\ntask.json| \"systemd:skladchik-reports-monitor-supervisor.timer\",\ntask.json| \"ssh:debian@[PRIVATE_IP]:/usr/local/sbin/skladchik-reports-monitor-cookie-update-edge\",\ntask.json| \"ssh:debian@[PRIVATE_IP]:/etc/skladchik-reports-monitor/secrets.env\",\ntask.json| \"ssh:debian@[PRIVATE_IP]:/var/lib/skladchik-reports-monitor/cookies.jar\",\ntask.json| \"ssh:debian@[PRIVATE_IP]:/var/lib/skladchik-reports-monitor/health.txt\",\ntask.json| \"/usr/local/sbin/skladchik-reports-monitor-cookie-update\",\ntask.json| \"/usr/local/sbin/skladchik-reports-monitor-backup\",\ntask.json| \"/usr/local/sbin/skladchik-reports-monitor-monthly-selftest-watch\",\ntask.json| \"/usr/local/sbin/skladchik-reports-monitor-status\",\ntask.json| \"/usr/local/sbin/skladchik-reports-monitor-supervisor\",\ntask.json| \"/srv/homelab-desired-state/sbin/skladchik-reports-monitor-backup\",\ntask.json| \"/var/lib/skladchik-reports-monitor-supervisor\"\ntask.json| ],\ntask.json| \"writes\": [\ntask.json| \"/var/lib/homelab-admin/seals/skladchik-reports-monitor-controlled-reauth-v1.json\",\ntask.json| \"systemd:skladchik-reports-monitor-supervisor.service\",\ntask.json| \"ssh:debian@[PRIVATE_IP]:/etc/skladchik-reports-monitor/secrets.env\",\ntask.json| \"ssh:debian@[PRIVATE_IP]:/var/lib/skladchik-reports-monitor/cookies.jar\",\ntask.json| \"ssh:debian@[PRIVATE_IP]:/var/lib/skladchik-reports-monitor/health.txt\",\ntask.json| \"ssh:debian@[PRIVATE_IP]:/var/lib/homelab-admin/rollback/skladchik-reports-monitor-controlled-reauth-v1\",\ntask.json| \"$HOMELAB_RUN_DIR/preflight.json\",\ntask.json| \"$HOMELAB_RUN_DIR/prepare.json\",\ntask.json| \"$HOMELAB_RUN_DIR/dry-run.json\",\ntask.json| \"$HOMELAB_RUN_DIR/apply.json\",\ntask.json| \"$HOMELAB_RUN_DIR/verify.json\",\ntask.json| \"$HOMELAB_RUN_DIR/rollback.json\",\ntask.json| \"$HOMELAB_RUN_DIR/seal.json\",\ntask.json| \"/usr/local/sbin/skladchik-reports-monitor-cookie-update\",\ntask.json| \"/usr/local/sbin/skladchik-reports-monitor-backup\",\ntask.json| \"/usr/local/sbin/skladchik-reports-monitor-monthly-selftest-watch\",\ntask.json| \"/usr/local/sbin/skladchik-reports-monitor-status\",\ntask.json| \"/usr/local/sbin/skladchik-reports-monitor-supervisor\",\ntask.json| \"/srv/homelab-desired-state/sbin/skladchik-reports-monitor-cookie-update\",\ntask.json| \"/srv/homelab-desired-state/sbin/skladchik-reports-monitor-backup\",\ntask.json| \"/srv/homelab-desired-state/sbin/skladchik-reports-monitor-monthly-selftest-watch\",\ntask.json| \"/srv/homelab-desired-state/sbin/skladchik-reports-monitor-status\",\ntask.json| \"/srv/homelab-desired-state/sbin/skladchik-reports-monitor-supervisor\",\ntask.json| \"/mnt/staging/homelab-admin-rollback/skladchik-reports-monitor-controlled-reauth-v1\",\ntask.json| \"ssh:debian@[PRIVATE_IP]:/usr/local/sbin/skladchik-reports-monitor-cookie-update-edge\"\ntask.json| ],\ntask.json| \"mutates\": [\ntask.json| \"systemd:skladchik-reports-monitor-supervisor.service\",\ntask.json| \"ssh:debian@[PRIVATE_IP]:/etc/skladchik-reports-monitor/secrets.env\",\ntask.json| \"ssh:debian@[PRIVATE_IP]:/var/lib/skladchik-reports-monitor/cookies.jar\",\ntask.json| \"ssh:debian@[PRIVATE_IP]:systemd:skladchik-reports-monitor.service\",\ntask.json| \"ssh:debian@[PRIVATE_IP]:systemd:skladchik-reports-monitor.timer\",\ntask.json| \"/usr/local/sbin/skladchik-reports-monitor-cookie-update\",\ntask.json| \"/usr/local/sbin/skladchik-reports-monitor-backup\",\ntask.json| \"/usr/local/sbin/skladchik-reports-monitor-monthly-selftest-watch\",\ntask.json| \"/usr/local/sbin/skladchik-reports-monitor-status\",\ntask.json| \"/usr/local/sbin/skladchik-reports-monitor-supervisor\",\ntask.json| \"git:/srv/homelab-desired-state\",\ntask.json| \"ssh:debian@[PRIVATE_IP]:/usr/local/sbin/skladchik-reports-monitor-cookie-update-edge\"\ntask.json| ],\ntask.json| \"expected_outputs\": [\ntask.json| {\ntask.json| \"path\": \"$HOMELAB_RUN_DIR/preflight.json\",\ntask.json| \"producer\": \"phases/preflight.sh\",\ntask.json| \"verification\": \"strict host-key access, the pinned live predecessor edge script, the versioned target edge script, AUTH_REQUIRED and bounded monitor quiescence are confirmed; evidence records both SHAs\"\ntask.json| },\ntask.json| {\ntask.json| \"path\": \"$HOMELAB_RUN_DIR/prepare.json\",\ntask.json| \"producer\": \"phases/prepare.sh\",\ntask.json| \"verification\": \"root-only local and edge rollback snapshots have self-contained SHA256 manifests\"\ntask.json| },\ntask.json| {\ntask.json| \"path\": \"$HOMELAB_RUN_DIR/dry-run.json\",\ntask.json| \"producer\": \"phases/dry-run.sh\",\ntask.json| \"verification\": \"rollback manifests validate, the pre-apply live edge script remains the pinned predecessor, the versioned deployment asset matches the pinned target SHA256, and evidence records both SHAs\"\ntask.json| },\ntask.json| {\ntask.json| \"path\": \"$HOMELAB_RUN_DIR/apply.json\",\ntask.json| \"producer\": \"phases/apply.sh\",\ntask.json| \"verification\": \"apply replaces the pinned predecessor edge script with the pinned target script, quiesces the timer, performs one controlled reauthentication and restores or enables the timer\"\ntask.json| },\ntask.json| {\ntask.json| \"path\": \"$HOMELAB_RUN_DIR/verify.json\",\ntask.json| \"producer\": \"phases/verify.sh\",\ntask.json| \"verification\": \"edge monitor, counts, timer, supervisor, backup and restore are OK\"\ntask.json| },\ntask.json| {\ntask.json| \"path\": \"$HOMELAB_RUN_DIR/rollback.json\",\ntask.json| \"producer\": \"phases/rollback.sh\",\ntask.json| \"verification\": \"previous auth artifacts and timer state are restored\"\ntask.json| },\ntask.json| {\ntask.json| \"path\": \"$HOMELAB_RUN_DIR/seal.json\",\ntask.json| \"producer\": \"phases/seal.sh\",\ntask.json| \"verification\": \"standard seal exists and rollback snapshot is removed\"\ntask.json| }\ntask.json| ],\ntask.json| \"runtime_matrix\": [\ntask.json| {\ntask.json| \"host\": \"pve01.gram1.ru\",\ntask.json| \"runtime\": \"bash,jq,git,systemd,ssh\",\ntask.json| \"compile\": \"bash -n; jq -e\"\ntask.json| },\ntask.json| {\ntask.json| \"host\": \"edge\",\ntask.json| \"runtime\": \"bash,python3,systemd,sha256sum\",\ntask.json| \"compile\": \"bash -n\"\ntask.json| }\ntask.json| ],\ntask.json| \"rollback\": {\ntask.json| \"command\": \"HOMELAB_CONTROLLED_APPLY_CONFIRM=skladchik-reports-monitor-controlled-reauth-v1 /opt/homelab-admin/bin/homelab-admin run TASK_DIR rollback\",\ntask.json| \"verification\": \"self-contained local and edge manifests validate; prior auth artifacts, scripts, desired-state HEAD, supervisor state and timer state are restorable\"\ntask.json| },\ntask.json| \"no_change_assertions\": [\ntask.json| \"vm160_backup_job\",\ntask.json| \"cluster_backup_jobs\",\ntask.json| \"unrelated_guests\",\ntask.json| \"network\",\ntask.json| \"dns\",\ntask.json| \"known_hosts\",\ntask.json| \"mail_cloud\",\ntask.json| \"gitea\",\ntask.json| \"backup_and_restore_artifacts\"\ntask.json| ],\ntask.json| \"source_of_truth\": {\ntask.json| \"change_id\": \"CR-2026-0012\",\ntask.json| \"incident\": \"Skladchik monitor AUTH_REQUIRED with missing complaint counts\",\ntask.json| \"edge_script_sha256\": \"10fe854274dca1a053cfd8c0155340bd689bf3f6555c70dbc14b890cbafb93fa\",\ntask.json| \"edge_script_path\": \"/usr/local/sbin/skladchik-reports-monitor-cookie-update-edge\",\ntask.json| \"unsafe_legacy_wrapper_path\": \"/usr/local/sbin/skladchik-reports-monitor-cookie-update\",\ntask.json| \"unsafe_legacy_wrapper_reason\": \"legacy host-key auto-acceptance\",\ntask.json| \"strict_hostkey_required\": true,\ntask.json| \"hidden_interactive_prompt_required\": true,\ntask.json| \"cookie_content_must_not_enter_git_or_logs\": true,\ntask.json| \"backup_status_before\": \"OK\",\ntask.json| \"restore_status_before\": \"OK\",\ntask.json| \"regression_lock\": \"REG-024-SKLADCHIK-REAUTH-STRICT-HOSTKEY\",\ntask.json| \"active_pve01_script_count\": 5,\ntask.json| \"active_pve01_original_sha256\": {\ntask.json| \"skladchik-reports-monitor-cookie-update\": \"a3006723d902fb7735c593003a6c6755fa3d69683126ec5f53165131f754fd7a\",\ntask.json| \"skladchik-reports-monitor-backup\": \"ad0652d1e154fca94a20bdf831b40b831fd0d862eb104ec3377f486e96270d0a\",\ntask.json| \"skladchik-reports-monitor-monthly-selftest-watch\": \"dd2c7e26aaa6835fa4ac4365eade338eb7b19f4bfe7528179c59cf8039454180\",\ntask.json| \"skladchik-reports-monitor-status\": \"f94ec75155f415b35517dade0bfba00926eba6088e0c2b5c96ae79daa5dc368f\",\ntask.json| \"skladchik-reports-monitor-supervisor\": \"69494a4c32fa632844cdb54aabbe302a14d478924efcb719c6cd3d594d102369\"\ntask.json| },\ntask.json| \"active_pve01_deployed_sha256\": {\ntask.json| \"skladchik-reports-monitor-cookie-update\": \"ea2d89ce67893a6d89b29610e4d46e207932ecc157173454585f67890f58810c\",\ntask.json| \"skladchik-reports-monitor-backup\": \"1912f8f0559637563972bb8ac984fb0036694092056ae53968c02cc3b67855f3\",\ntask.json| \"skladchik-reports-monitor-monthly-selftest-watch\": \"48c6ca01d2430d14c6af797c079640815ba1352a98466d4014ecbb80a27eaada\",\ntask.json| \"skladchik-reports-monitor-status\": \"f902a61109efbac5918360f2235c2523bd10704ac484a4e8591e8a1c490faf5a\",\ntask.json| \"skladchik-reports-monitor-supervisor\": \"35c68ae0929ff3795be2bde40de334fc8e26413a3d8a620a137be90d1f2c164e\"\ntask.json| },\ntask.json| \"edge_mutation_script_versioned\": true,\ntask.json| \"desired_state_script_count_after_apply\": 5,\ntask.json| \"all_active_skladchik_ssh_requires_strict_known_hosts\": true,\ntask.json| \"unsafe_active_hostkey_markers_after_verify\": 0,\ntask.json| \"pve01_assets_checked_by_repository_tests\": true,\ntask.json| \"regression_locks\": [\ntask.json| \"REG-024-SKLADCHIK-REAUTH-STRICT-HOSTKEY\",\ntask.json| \"REG-025-EXACT-SHA256-LENGTH-VALIDATION\",\ntask.json| \"REG-026-ACTIVE-SKLADCHIK-STRICT-HOSTKEY-COVERAGE\",\ntask.json| \"REG-027-REMOTE-MUTATION-SCRIPT-MUST-BE-VERSIONED\",\ntask.json| \"REG-028-FORBIDDEN-TOKEN-IN-NEGATIVE-ASSERTION\",\ntask.json| \"REG-029-LEGACY-WRAPPER-NOT-COPIED-VERBATIM\",\ntask.json| \"REG-030-BINARY-ARTIFACT-TEXT-SCAN\",\ntask.json| \"REG-031-SEMANTIC-TEST-LOOP-EXPANSION-MISMATCH\",\ntask.json| \"REG-032-OVERBROAD-INLINE-PYTHON-TEST\",\ntask.json| \"REG-033-INSTALLED-SOURCE-MUST-BE-DECLARED\",\ntask.json| \"REG-034-ROLLBACK-STAGE-PARENT-MODE\",\nRUN_EVIDENCE_END\nCURRENT_FACTS_BEGIN\nEDGE_SCRIPT_LOCAL_EXISTS=true\nEDGE_SCRIPT_LOCAL_META=mode=700|uid=0|gid=0|size=5247\nEDGE_SCRIPT_LOCAL_SHA256=10fe854274dca1a053cfd8c0155340bd689bf3f6555c70dbc14b890cbafb93fa\nLOCAL_HEALTH_EXISTS=false\nSECRETS_ENV_EXISTS=false\nMONITOR_PY_EXISTS=false\nUNIT=skladchik-reports-monitor.service|active=inactive|enabled=not-found\nUNIT=skladchik-reports-monitor.timer|active=inactive|enabled=not-found\nUNIT=skladchik-reports-monitor-supervisor.service|active=failed|enabled=static\nUNIT=skladchik-reports-monitor-supervisor.timer|active=active|enabled=enabled\nCURRENT_FACTS_END\nRCA_CAPTURE_COMPLETE=true\nCHANGES_MADE_BY_943=false\nROLLBACK_STARTED=false\nROLLBACK_RESTORED=null\nHOMELAB_RESULT_CONTRACT={\"version\":1,\"command_id\":\"SUPPORT-260923-SKLADCHIK-PREFLIGHT-LINE13-RCA-943R1\",\"status\":\"OK\",\"changes_made\":false,\"rollback_started\":false,\"rollback_restored\":null}\nSKLADCHIK943_END=true\n" } diff --git a/runtime/latest.txt b/runtime/latest.txt index 3c3ac099..6893533f 100644 --- a/runtime/latest.txt +++ b/runtime/latest.txt @@ -1,121 +1,349 @@ CHAT_OUTPUT_BEGIN -COMMAND_ID=SUPPORT-260923-SKLADCHIK-COOKIE-REFRESH-REISSUE-942R1 -STATUS=FAIL -RC=1 +COMMAND_ID=SUPPORT-260923-SKLADCHIK-PREFLIGHT-LINE13-RCA-943R1 +STATUS=OK +RC=0 HOST=pve01 -MODE=change -COMPONENT=skladchik-cookie-refresh-reissue +MODE=read-only +COMPONENT=skladchik-preflight-line13-rca REFERENCE_REGISTER_CHECK=OK REFERENCE_SHA256=5a3d8e5154c41cb582a4e0aca68090be1f0138918bf82131a948df326f9d8d66 ERROR_REGISTER_CHECK=OK ERROR_REGISTER_SHA256=3b09a553ec0f527ed3afeed4753f52a74ee3036045bbfb3c685e9f8af4ba7ba0 -COMMAND_SHA256=27da7524dbfc342b7925b1034305a552a30bcb635ca4e09e5ee7c3bfdb3e51b6 +COMMAND_SHA256=dd3a88084befa0f31cc5f49ef676cb01eec2dfd507c1e1bfd993fba637e753d7 DUPLICATE_FAILED_COMMAND_BLOCKED=false EXECUTION_STARTED=true -CHANGE_DECLARED=true +CHANGE_DECLARED=false RESULT_CONTRACT_VALID=true -RESULT_CONTRACT_STATUS=FAIL +RESULT_CONTRACT_STATUS=NOT_APPLICABLE RESULT_CONTRACT_ERROR=NONE -COMMAND_RC=1 -CHANGES_MADE=true +COMMAND_RC=0 +CHANGES_MADE=false ROLLBACK_STARTED=false ROLLBACK_RESTORED=null -MUTATION_OUTCOME=MUTATED +MUTATION_OUTCOME=NO_MUTATION SANITIZED=yes SECRETS_INCLUDED=no PRIVATE_ADDRESSES_INCLUDED=no -RAW_EVIDENCE_SHA256=edad064e291af1b094dd2a4d47ae4372a5fa3ddfdb4518232751e42562c271f5 -SANITIZED_OUTPUT_SHA256=a51b6642d8cde7b817b5bfd3ee7d5b5a3b22459a9dee1f3d368a3ef292aebb3e +RAW_EVIDENCE_SHA256=2c42ff80da827c41e303d1369683a91c76fe254e8a5cc304eeb6038334a8dafa +SANITIZED_OUTPUT_SHA256=17ace4d0e724d6b41d9460dad37dfe85e5b452f724ea763d0ba457d6ad255c77 OUTPUT_BEGIN -SKLADCHIK942_BEGIN=true -COMMAND_ID=SUPPORT-260923-SKLADCHIK-COOKIE-REFRESH-REISSUE-942R1 -MODE=change -COMPONENT=skladchik-cookie-refresh-reissue -PREDECESSOR_929=PASS_SKLADCHIK_929_TRANSPORT_RECONCILE_SAFE_TO_REISSUE -PREDECESSOR_931=LOCAL_TRANSPORT_RC126_NO_RUNNER_RESULT -PREDECESSOR_932=PASS_SKLADCHIK_931_LOCAL_CARRIER_FORENSICS -PREDECESSOR_933=HOLD_SKLADCHIK_933_COOKIE_STDIN_INVALID_NO_MUTATION -PREDECESSOR_934=HOLD_SKLADCHIK_934_COOKIE_STDIN_INVALID_NO_MUTATION -931_RCA=UPLOAD_SUCCEEDED;DIRECT_EXEC_FROM_RUN_RETURNED_126;STDERR_63_BYTES_MATCHES_BASH_PERMISSION_DENIED;FIX_RUN_OUTER_VIA_USR_BIN_BASH -933_RCA=RUNNER_REACHED;COOKIE_CLIPBOARD_STDIN_INVALID;NO_MUTATION -934_RCA=RUNNER_REACHED;READ_SUCCEEDED_BUT_COOKIE_VALIDATION_FAILED;NO_MUTATION;COMMON_FAILURE_IS_WINDOWS_INPUT_TRANSPORT -935_TRANSPORT_PROOF=MASKED_GUI_TEXTBOX_WITH_PASTE;DIRECT_WRITE_TO_PLINK_PROCESS_STANDARDINPUT;COOKIE_INPUT_LENGTH_325_REACHED_RUNNER -PREDECESSOR_935=OFFICIAL_REFRESH_REACHED_INITIAL_STATE_CHECK;FAILED_BEFORE_COOKIE_PROMPT -PREDECESSOR_939=ONLY_INITIAL_CHECK_LINE_159_FAILED -PREDECESSOR_940=LOCAL_BEHIND_FAST_FORWARD_TO_ORIGIN -PREDECESSOR_941=PASS_HOMELAB_OPS_FASTFORWARD;INITIAL_STATE_CHECK_ALL_PASS -942_PURPOSE=REISSUE_OFFICIAL_COOKIE_REFRESH_AFTER_LINE_159_REMEDIATION -MUTATION_BOUNDARY=OFFICIAL_SKLADCHIK_COOKIE_REFRESH_ONLY;TRANSIENT_ROOT_ONLY_COOKIE_STAGING;NO_OTHER_SERVICES;NO_VM_CONFIG;NO_CLOUD_CONFIG -COOKIE_INPUT_LENGTH=325 -INITIAL_STATE_GIT_GATE=PASS -RUNNER_FIRST_COOKIE_INPUT=true -OFFICIAL_RUNBOOK_VERIFIED=true -COOKIE_SOURCE=WINDOWS_CHROME_MASKED_GUI_DIRECT_PROCESS_STANDARDINPUT -COOKIE_SECRET_OUTPUT=NONE -SCOPE=SKLADCHIK_COOKIE_REFRESH_VALIDATE_PREFLIGHT_PREPARE_DRY_RUN_HIDDEN_COOKIE_APPLY_VERIFY_SEAL -ROLLBACK=AUTOMATIC_VERSIONED_RECOVERY_WORKFLOW_RESTORES_PREVIOUS_SECRET_COOKIE_JAR_SCRIPTS_DESIRED_STATE_TIMER_AND_SUPERVISOR_STATE_ON_FAILURE_BEFORE_SEAL -VERIFY=FRESH_COOKIE_FORCED_STALE_JAR_BYPASSED_SECOND_JAR_ONLY_PROBE_MONITOR_OK_NUMERIC_COUNTS_TARGET_EDGE_SHA_TIMERS_SUPERVISOR_SEAL_STAGE_REMOVAL_VM160_UNCHANGED_NO_CLOUD_COMMANDS -LINT_STATUS=OK -TASK_ID=skladchik-reports-monitor-controlled-reauth-v1 -POLICY_ID=homelab-admin-guardrails-v1.2 -RULES_CHECKED=14 -SOURCE_FILES_CHECKED=19 -TASK_VALID=YES -TASK_ID=skladchik-reports-monitor-controlled-reauth-v1 -TASK_SCOPE=controlled_apply -PRODUCTION_CHANGE=YES -LINT_STATUS=OK -TASK_ID=skladchik-reports-monitor-controlled-reauth-v1 -POLICY_ID=homelab-admin-guardrails-v1.2 -RULES_CHECKED=14 -SOURCE_FILES_CHECKED=19 -ROOT_CAUSE=UNEXPECTED_PHASE_ERROR -ROOT_CAUSE_COUNT=1 -DETAIL=phase=preflight line=13 rc=1 -RUN_DIR=/var/lib/homelab-admin/runs/skladchik-reports-monitor-controlled-reauth-v1/20260923T120316Z-986653-preflight -RUN_RC=1 -[1;35m===== СКОПИРОВАТЬ МНЕ В ЧАТ ===== -COOKIE_REFRESH_RESULT=FAILED -FAILED_STAGE=PREFLIGHT -FAILED_RC=1 -ROLLBACK_ATTEMPTED=NO -ROLLBACK_RC=NOT_RUN -SEALED=1 -BACKUP_WAIT_EVENTS=0 -ACTIVE_BACKUP_RESTORE_NOW=0 -APPLY_RUN=unknown -COOKIE_VALUES_PRINTED=NO -VM160_CHANGED=NO -CLOUD_COMMANDS_EXECUTED=NO -NEXT_PHASE=PASTE_THIS_FRAGMENT_ONLY -===== КОНЕЦ ФРАГМЕНТА =====[0m -PTY_COOKIE_FEED_SENT=false -PTY_COOKIE_PROMPT_SEEN=false -OFFICIAL_REFRESH_RC=1 -PTY_COOKIE_FEED_SENT=false -PTY_COOKIE_PROMPT_SEEN=false -COOKIE_REFRESH_RESULT_SUCCESS=false -FRESH_COOKIE_FORCED=false -STALE_COOKIE_JAR_BYPASSED=false -POST_SUPERVISOR_STATUS=FAIL -POST_MONITOR_STATUS=ERROR -POST_MONITOR_REPORTS=missing -POST_SESSION_KEEPALIVE=persistent_cookie_jar -POST_COOKIE_EXPIRY_STATUS=WARN -POST_COOKIE_JAR_PRESENT=false -POST_COOKIE_JAR_BYTES=0 -POST_NETBIRD_ROUTE_WT0=false -POST_ROUTE_PROBLEM_REPORTED=true -FAILURE=OFFICIAL_REFRESH_RC_1 -COOKIE_REFRESH_COMPLETE=false -DECISION=HOLD_SKLADCHIK_COOKIE_REFRESH_RCA -NEXT_GATE=SKLADCHIK_COOKIE_REFRESH_RCA -TASK_RESULT=HOLD_SKLADCHIK_COOKIE_REFRESH -CHANGES_MADE_BY_942=true +SKLADCHIK943_BEGIN=true +COMMAND_ID=SUPPORT-260923-SKLADCHIK-PREFLIGHT-LINE13-RCA-943R1 +MODE=read-only +COMPONENT=skladchik-preflight-line13-rca +PREDECESSOR_942=FAILED_STAGE_PREFLIGHT;DETAIL_phase_preflight_line_13_rc_1 +MUTATION_BOUNDARY=NONE_READ_ONLY +PHASE_PREFLIGHT_SOURCE_BEGIN +PHASE_PREFLIGHT_SOURCE_PRESENT=true +PHASE_PREFLIGHT_SOURCE_SHA256=f3e533ebd72ba2343a78797a8087c529c9fcfa2b3cabc45b3c0bd477356a8cc9 +PHASE_PREFLIGHT_SOURCE_BYTES=3076 +PHASE_PREFLIGHT_SOURCE_LINES=29 +0001:#!/usr/bin/env bash +0002:set -Eeuo pipefail +0003:on_err(){ local rc=$?; trap - ERR; printf 'ROOT_CAUSE=UNEXPECTED_PHASE_ERROR\nROOT_CAUSE_COUNT=1\nDETAIL=phase=preflight line=%s rc=%s\n' "${1:-0}" "$rc"; exit "$rc"; } +0004:trap 'on_err "$LINENO"' ERR +0005:export SYSTEMD_PAGER=cat PAGER=cat SYSTEMD_LESS=FRXMK +0006:TASK_ID=skladchik-reports-monitor-controlled-reauth-v1 +0007:EDGE=debian@[PRIVATE_IP] +0008:CURRENT_EDGE_SHA=b13d846cf089d450e3d5ac9e85ad1e232c198930e466bb8b82f6983a4d48ce0e +0009:TARGET_EDGE_SHA=10fe854274dca1a053cfd8c0155340bd689bf3f6555c70dbc14b890cbafb93fa +0010:SSH=(-o BatchMode=yes -o ConnectTimeout=12 -o StrictHostKeyChecking=yes -o UserKnownHostsFile=[SENSITIVE_PATH] +0011:test "$(hostname -f)" = pve01.gram1.ru +0012:test -z "$(git -C /srv/homelab-desired-state status --porcelain)" +0013:test "$(sha256sum /usr/local/sbin/skladchik-reports-monitor-cookie-update | awk '{print $1}')" = a3006723d902fb7735c593003a6c6755fa3d69683126ec5f53165131f754fd7a +0014:test "$(sha256sum /usr/local/sbin/skladchik-reports-monitor-backup | awk '{print $1}')" = ad0652d1e154fca94a20bdf831b40b831fd0d862eb104ec3377f486e96270d0a +0015:test "$(sha256sum /usr/local/sbin/skladchik-reports-monitor-monthly-selftest-watch | awk '{print $1}')" = dd2c7e26aaa6835fa4ac4365eade338eb7b19f4bfe7528179c59cf8039454180 +0016:test "$(sha256sum /usr/local/sbin/skladchik-reports-monitor-status | awk '{print $1}')" = f94ec75155f415b35517dade0bfba00926eba6088e0c2b5c96ae79daa5dc368f +0017:test "$(sha256sum /usr/local/sbin/skladchik-reports-monitor-supervisor | awk '{print $1}')" = 69494a4c32fa632844cdb54aabbe302a14d478924efcb719c6cd3d594d102369 +0018:test "$(sha256sum /srv/homelab-desired-state/sbin/skladchik-reports-monitor-backup | awk '{print $1}')" = ad0652d1e154fca94a20bdf831b40b831fd0d862eb104ec3377f486e96270d0a +0019:test "$(sha256sum "$HOMELAB_TASK_DIR/assets/edge/skladchik-reports-monitor-cookie-update-edge" | awk '{print $1}')" = "$TARGET_EDGE_SHA" +0020:RESULT="$(ssh "${SSH[@]}" "$EDGE" sudo -n /bin/bash -s -- "$CURRENT_EDGE_SHA" < "$HOMELAB_TASK_DIR/assets/edge-preflight.sh")" +0021:grep -Fq 'EDGE_PREFLIGHT_STATUS=OK' <<<"$RESULT" +0022:test "$(systemctl is-enabled skladchik-reports-monitor-supervisor.timer)" = enabled +0023:test "$(systemctl is-active skladchik-reports-monitor-supervisor.timer)" = active +0024:test "$(systemctl show skladchik-reports-monitor-supervisor.service -p Result --value)" = exit-code +0025:grep -q '^STATUS=OK$' /var/lib/skladchik-reports-monitor-backup/health.txt +0026:grep -q '^STATUS=OK$' /var/lib/skladchik-reports-monitor-backup/restore-check-health.txt +0027:jq -n --arg status OK --arg current_edge_script_sha "$CURRENT_EDGE_SHA" --arg target_edge_script_sha "$TARGET_EDGE_SHA" '{status:$status,root_cause_count:0,production_changed:false,edge_monitor_status:"AUTH_REQUIRED",current_edge_script_sha256:$current_edge_script_sha,target_edge_script_sha256:$target_edge_script_sha,strict_hostkey:true,active_skladchik_sources_pinned:true}' > "$HOMELAB_RUN_DIR/preflight.json" +0028:printf '%s\n' "$RESULT" +0029:printf 'PREFLIGHT_STATUS=OK\nACTIVE_SKLADCHIK_SOURCE_HASHES=PINNED\nEDGE_MUTATION_SCRIPT=VERSIONED\nROOT_CAUSE_COUNT=0\nPRODUCTION_CHANGED=NO\nREAUTH_EXECUTED=NO\n' +PHASE_PREFLIGHT_SOURCE_END +EDGE_PREFLIGHT_SOURCE_BEGIN +EDGE_PREFLIGHT_SOURCE_PRESENT=true +EDGE_PREFLIGHT_SOURCE_SHA256=77ebb2893962550f97d80bdfd2eb414f666578a272b6c35a3f4ca669df117878 +EDGE_PREFLIGHT_SOURCE_BYTES=1351 +EDGE_PREFLIGHT_SOURCE_LINES=29 +0001:#!/usr/bin/env bash +0002:set -Eeuo pipefail +0003:EXPECTED_SHA="${1:?expected script sha required}" +0004:SCRIPT=/usr/local/sbin/skladchik-reports-monitor-cookie-update-edge +0005:HEALTH=/var/lib/skladchik-reports-monitor/health.txt +0006:test -x "$SCRIPT" +0007:test "$(sha256sum "$SCRIPT" | awk '{print $1}')" = "$EXPECTED_SHA" +0008:test -s /etc/skladchik-reports-monitor/secrets.env +0009:test -s /opt/skladchik-reports-monitor/monitor.py +0010:test -s "$HEALTH" +0011:STATUS="$(awk -F= '$1=="STATUS"{print $2;exit}' "$HEALTH")" +0012:test "$STATUS" = AUTH_REQUIRED +0013:test "$(systemctl is-enabled skladchik-reports-monitor.timer)" = enabled +0014:test "$(systemctl is-active skladchik-reports-monitor.timer)" = active +0015:WAIT_SECONDS=90 +0016:DEADLINE=$((SECONDS+WAIT_SECONDS)) +0017:monitor_busy() { +0018: systemctl is-active --quiet skladchik-reports-monitor.service || +0019: pgrep -f '/opt/skladchik-reports-monitor/monitor.py' >/dev/null || +0020: pgrep -f 'skladchik-reports-monitor-cookie-update-edge' >/dev/null +0021:} +0022:while monitor_busy; do +0023: if (( SECONDS >= DEADLINE )); then +0024: printf 'EDGE_PREFLIGHT_STATUS=BUSY\nFAILED_PHASE=WAIT_MONITOR_QUIESCENCE\nWAIT_SECONDS=%s\n' "$WAIT_SECONDS" +0025: exit 75 +0026: fi +0027: sleep 2 +0028:done +0029:printf 'EDGE_PREFLIGHT_STATUS=OK\nEDGE_MONITOR_STATUS=%s\nEDGE_SCRIPT_SHA256=%s\nEDGE_TIMER_ENABLED=YES\nEDGE_TIMER_ACTIVE=YES\nMONITOR_QUIESCENCE=OK\nWAIT_LIMIT_SECONDS=%s\n' "$STATUS" "$EXPECTED_SHA" "$WAIT_SECONDS" +EDGE_PREFLIGHT_SOURCE_END +FAILED_LINE_CONTEXT_BEGIN +PHASE_LINE_0009=TARGET_EDGE_SHA=10fe854274dca1a053cfd8c0155340bd689bf3f6555c70dbc14b890cbafb93fa +PHASE_LINE_0010=SSH=(-o BatchMode=yes -o ConnectTimeout=12 -o StrictHostKeyChecking=yes -o UserKnownHostsFile=[SENSITIVE_PATH] +PHASE_LINE_0011=test "$(hostname -f)" = pve01.gram1.ru +PHASE_LINE_0012=test -z "$(git -C /srv/homelab-desired-state status --porcelain)" +PHASE_LINE_0013=test "$(sha256sum /usr/local/sbin/skladchik-reports-monitor-cookie-update | awk '{print $1}')" = a3006723d902fb7735c593003a6c6755fa3d69683126ec5f53165131f754fd7a +PHASE_LINE_0014=test "$(sha256sum /usr/local/sbin/skladchik-reports-monitor-backup | awk '{print $1}')" = ad0652d1e154fca94a20bdf831b40b831fd0d862eb104ec3377f486e96270d0a +PHASE_LINE_0015=test "$(sha256sum /usr/local/sbin/skladchik-reports-monitor-monthly-selftest-watch | awk '{print $1}')" = dd2c7e26aaa6835fa4ac4365eade338eb7b19f4bfe7528179c59cf8039454180 +PHASE_LINE_0016=test "$(sha256sum /usr/local/sbin/skladchik-reports-monitor-status | awk '{print $1}')" = f94ec75155f415b35517dade0bfba00926eba6088e0c2b5c96ae79daa5dc368f +PHASE_LINE_0017=test "$(sha256sum /usr/local/sbin/skladchik-reports-monitor-supervisor | awk '{print $1}')" = 69494a4c32fa632844cdb54aabbe302a14d478924efcb719c6cd3d594d102369 +EDGE_LINE_0009=test -s /opt/skladchik-reports-monitor/monitor.py +EDGE_LINE_0010=test -s "$HEALTH" +EDGE_LINE_0011=STATUS="$(awk -F= '$1=="STATUS"{print $2;exit}' "$HEALTH")" +EDGE_LINE_0012=test "$STATUS" = AUTH_REQUIRED +EDGE_LINE_0013=test "$(systemctl is-enabled skladchik-reports-monitor.timer)" = enabled +EDGE_LINE_0014=test "$(systemctl is-active skladchik-reports-monitor.timer)" = active +EDGE_LINE_0015=WAIT_SECONDS=90 +EDGE_LINE_0016=DEADLINE=$((SECONDS+WAIT_SECONDS)) +EDGE_LINE_0017=monitor_busy() { +FAILED_LINE_CONTEXT_END +RUN_EVIDENCE_BEGIN +RUN_FILE=stdout.log|present=true|bytes=89 +stdout.log|ROOT_CAUSE=UNEXPECTED_PHASE_ERROR +stdout.log|ROOT_CAUSE_COUNT=1 +stdout.log|DETAIL=phase=preflight line=13 rc=1 +RUN_FILE=stderr.log|present=true|bytes=0 +RUN_FILE=preflight.json|present=false|bytes=0 +RUN_FILE=result.json|present=true|bytes=592 +result.json|{ +result.json| "run_id": "20260923T120316Z-986653-preflight", +result.json| "task_id": "skladchik-reports-monitor-controlled-reauth-v1", +result.json| "phase": "preflight", +result.json| "mutation_scope": "controlled_apply", +result.json| "production_change": true, +result.json| "started_at": "2026-09-23T12:03:16Z", +result.json| "finished_at": "2026-09-23T12:03:16Z", +result.json| "status": "FAILED", +result.json| "rc": 1, +result.json| "stdout": "/var/lib/homelab-admin/runs/skladchik-reports-monitor-controlled-reauth-v1/20260923T120316Z-986653-preflight/stdout.log", +result.json| "stderr": "/var/lib/homelab-admin/runs/skladchik-reports-monitor-controlled-reauth-v1/20260923T120316Z-986653-preflight/stderr.log" +result.json|} +RUN_FILE=task.json|present=true|bytes=13579 +task.json|{ +task.json| "schema_version": 2, +task.json| "task_id": "skladchik-reports-monitor-controlled-reauth-v1", +task.json| "target_host": "pve01.gram1.ru", +task.json| "mutation_scope": "controlled_apply", +task.json| "production_change": true, +task.json| "failure_mode": "fail_fast", +task.json| "guards": { +task.json| "desired_state_clean": true, +task.json| "failed_units_max": 4, +task.json| "active_backup_restore_max": 0 +task.json| }, +task.json| "reads": [ +task.json| "[SENSITIVE_PATH] +task.json| "/var/lib/skladchik-reports-monitor-backup/health.txt", +task.json| "/var/lib/skladchik-reports-monitor-backup/restore-check-health.txt", +task.json| "systemd:skladchik-reports-monitor-supervisor.service", +task.json| "systemd:skladchik-reports-monitor-supervisor.timer", +task.json| "ssh:debian@[PRIVATE_IP]:/usr/local/sbin/skladchik-reports-monitor-cookie-update-edge", +task.json| "ssh:debian@[PRIVATE_IP]:/etc/skladchik-reports-monitor/secrets.env", +task.json| "ssh:debian@[PRIVATE_IP]:/var/lib/skladchik-reports-monitor/cookies.jar", +task.json| "ssh:debian@[PRIVATE_IP]:/var/lib/skladchik-reports-monitor/health.txt", +task.json| "/usr/local/sbin/skladchik-reports-monitor-cookie-update", +task.json| "/usr/local/sbin/skladchik-reports-monitor-backup", +task.json| "/usr/local/sbin/skladchik-reports-monitor-monthly-selftest-watch", +task.json| "/usr/local/sbin/skladchik-reports-monitor-status", +task.json| "/usr/local/sbin/skladchik-reports-monitor-supervisor", +task.json| "/srv/homelab-desired-state/sbin/skladchik-reports-monitor-backup", +task.json| "/var/lib/skladchik-reports-monitor-supervisor" +task.json| ], +task.json| "writes": [ +task.json| "/var/lib/homelab-admin/seals/skladchik-reports-monitor-controlled-reauth-v1.json", +task.json| "systemd:skladchik-reports-monitor-supervisor.service", +task.json| "ssh:debian@[PRIVATE_IP]:/etc/skladchik-reports-monitor/secrets.env", +task.json| "ssh:debian@[PRIVATE_IP]:/var/lib/skladchik-reports-monitor/cookies.jar", +task.json| "ssh:debian@[PRIVATE_IP]:/var/lib/skladchik-reports-monitor/health.txt", +task.json| "ssh:debian@[PRIVATE_IP]:/var/lib/homelab-admin/rollback/skladchik-reports-monitor-controlled-reauth-v1", +task.json| "$HOMELAB_RUN_DIR/preflight.json", +task.json| "$HOMELAB_RUN_DIR/prepare.json", +task.json| "$HOMELAB_RUN_DIR/dry-run.json", +task.json| "$HOMELAB_RUN_DIR/apply.json", +task.json| "$HOMELAB_RUN_DIR/verify.json", +task.json| "$HOMELAB_RUN_DIR/rollback.json", +task.json| "$HOMELAB_RUN_DIR/seal.json", +task.json| "/usr/local/sbin/skladchik-reports-monitor-cookie-update", +task.json| "/usr/local/sbin/skladchik-reports-monitor-backup", +task.json| "/usr/local/sbin/skladchik-reports-monitor-monthly-selftest-watch", +task.json| "/usr/local/sbin/skladchik-reports-monitor-status", +task.json| "/usr/local/sbin/skladchik-reports-monitor-supervisor", +task.json| "/srv/homelab-desired-state/sbin/skladchik-reports-monitor-cookie-update", +task.json| "/srv/homelab-desired-state/sbin/skladchik-reports-monitor-backup", +task.json| "/srv/homelab-desired-state/sbin/skladchik-reports-monitor-monthly-selftest-watch", +task.json| "/srv/homelab-desired-state/sbin/skladchik-reports-monitor-status", +task.json| "/srv/homelab-desired-state/sbin/skladchik-reports-monitor-supervisor", +task.json| "/mnt/staging/homelab-admin-rollback/skladchik-reports-monitor-controlled-reauth-v1", +task.json| "ssh:debian@[PRIVATE_IP]:/usr/local/sbin/skladchik-reports-monitor-cookie-update-edge" +task.json| ], +task.json| "mutates": [ +task.json| "systemd:skladchik-reports-monitor-supervisor.service", +task.json| "ssh:debian@[PRIVATE_IP]:/etc/skladchik-reports-monitor/secrets.env", +task.json| "ssh:debian@[PRIVATE_IP]:/var/lib/skladchik-reports-monitor/cookies.jar", +task.json| "ssh:debian@[PRIVATE_IP]:systemd:skladchik-reports-monitor.service", +task.json| "ssh:debian@[PRIVATE_IP]:systemd:skladchik-reports-monitor.timer", +task.json| "/usr/local/sbin/skladchik-reports-monitor-cookie-update", +task.json| "/usr/local/sbin/skladchik-reports-monitor-backup", +task.json| "/usr/local/sbin/skladchik-reports-monitor-monthly-selftest-watch", +task.json| "/usr/local/sbin/skladchik-reports-monitor-status", +task.json| "/usr/local/sbin/skladchik-reports-monitor-supervisor", +task.json| "git:/srv/homelab-desired-state", +task.json| "ssh:debian@[PRIVATE_IP]:/usr/local/sbin/skladchik-reports-monitor-cookie-update-edge" +task.json| ], +task.json| "expected_outputs": [ +task.json| { +task.json| "path": "$HOMELAB_RUN_DIR/preflight.json", +task.json| "producer": "phases/preflight.sh", +task.json| "verification": "strict host-key access, the pinned live predecessor edge script, the versioned target edge script, AUTH_REQUIRED and bounded monitor quiescence are confirmed; evidence records both SHAs" +task.json| }, +task.json| { +task.json| "path": "$HOMELAB_RUN_DIR/prepare.json", +task.json| "producer": "phases/prepare.sh", +task.json| "verification": "root-only local and edge rollback snapshots have self-contained SHA256 manifests" +task.json| }, +task.json| { +task.json| "path": "$HOMELAB_RUN_DIR/dry-run.json", +task.json| "producer": "phases/dry-run.sh", +task.json| "verification": "rollback manifests validate, the pre-apply live edge script remains the pinned predecessor, the versioned deployment asset matches the pinned target SHA256, and evidence records both SHAs" +task.json| }, +task.json| { +task.json| "path": "$HOMELAB_RUN_DIR/apply.json", +task.json| "producer": "phases/apply.sh", +task.json| "verification": "apply replaces the pinned predecessor edge script with the pinned target script, quiesces the timer, performs one controlled reauthentication and restores or enables the timer" +task.json| }, +task.json| { +task.json| "path": "$HOMELAB_RUN_DIR/verify.json", +task.json| "producer": "phases/verify.sh", +task.json| "verification": "edge monitor, counts, timer, supervisor, backup and restore are OK" +task.json| }, +task.json| { +task.json| "path": "$HOMELAB_RUN_DIR/rollback.json", +task.json| "producer": "phases/rollback.sh", +task.json| "verification": "previous auth artifacts and timer state are restored" +task.json| }, +task.json| { +task.json| "path": "$HOMELAB_RUN_DIR/seal.json", +task.json| "producer": "phases/seal.sh", +task.json| "verification": "standard seal exists and rollback snapshot is removed" +task.json| } +task.json| ], +task.json| "runtime_matrix": [ +task.json| { +task.json| "host": "pve01.gram1.ru", +task.json| "runtime": "bash,jq,git,systemd,ssh", +task.json| "compile": "bash -n; jq -e" +task.json| }, +task.json| { +task.json| "host": "edge", +task.json| "runtime": "bash,python3,systemd,sha256sum", +task.json| "compile": "bash -n" +task.json| } +task.json| ], +task.json| "rollback": { +task.json| "command": "HOMELAB_CONTROLLED_APPLY_CONFIRM=skladchik-reports-monitor-controlled-reauth-v1 /opt/homelab-admin/bin/homelab-admin run TASK_DIR rollback", +task.json| "verification": "self-contained local and edge manifests validate; prior auth artifacts, scripts, desired-state HEAD, supervisor state and timer state are restorable" +task.json| }, +task.json| "no_change_assertions": [ +task.json| "vm160_backup_job", +task.json| "cluster_backup_jobs", +task.json| "unrelated_guests", +task.json| "network", +task.json| "dns", +task.json| "known_hosts", +task.json| "mail_cloud", +task.json| "gitea", +task.json| "backup_and_restore_artifacts" +task.json| ], +task.json| "source_of_truth": { +task.json| "change_id": "CR-2026-0012", +task.json| "incident": "Skladchik monitor AUTH_REQUIRED with missing complaint counts", +task.json| "edge_script_sha256": "10fe854274dca1a053cfd8c0155340bd689bf3f6555c70dbc14b890cbafb93fa", +task.json| "edge_script_path": "/usr/local/sbin/skladchik-reports-monitor-cookie-update-edge", +task.json| "unsafe_legacy_wrapper_path": "/usr/local/sbin/skladchik-reports-monitor-cookie-update", +task.json| "unsafe_legacy_wrapper_reason": "legacy host-key auto-acceptance", +task.json| "strict_hostkey_required": true, +task.json| "hidden_interactive_prompt_required": true, +task.json| "cookie_content_must_not_enter_git_or_logs": true, +task.json| "backup_status_before": "OK", +task.json| "restore_status_before": "OK", +task.json| "regression_lock": "REG-024-SKLADCHIK-REAUTH-STRICT-HOSTKEY", +task.json| "active_pve01_script_count": 5, +task.json| "active_pve01_original_sha256": { +task.json| "skladchik-reports-monitor-cookie-update": "a3006723d902fb7735c593003a6c6755fa3d69683126ec5f53165131f754fd7a", +task.json| "skladchik-reports-monitor-backup": "ad0652d1e154fca94a20bdf831b40b831fd0d862eb104ec3377f486e96270d0a", +task.json| "skladchik-reports-monitor-monthly-selftest-watch": "dd2c7e26aaa6835fa4ac4365eade338eb7b19f4bfe7528179c59cf8039454180", +task.json| "skladchik-reports-monitor-status": "f94ec75155f415b35517dade0bfba00926eba6088e0c2b5c96ae79daa5dc368f", +task.json| "skladchik-reports-monitor-supervisor": "69494a4c32fa632844cdb54aabbe302a14d478924efcb719c6cd3d594d102369" +task.json| }, +task.json| "active_pve01_deployed_sha256": { +task.json| "skladchik-reports-monitor-cookie-update": "ea2d89ce67893a6d89b29610e4d46e207932ecc157173454585f67890f58810c", +task.json| "skladchik-reports-monitor-backup": "1912f8f0559637563972bb8ac984fb0036694092056ae53968c02cc3b67855f3", +task.json| "skladchik-reports-monitor-monthly-selftest-watch": "48c6ca01d2430d14c6af797c079640815ba1352a98466d4014ecbb80a27eaada", +task.json| "skladchik-reports-monitor-status": "f902a61109efbac5918360f2235c2523bd10704ac484a4e8591e8a1c490faf5a", +task.json| "skladchik-reports-monitor-supervisor": "35c68ae0929ff3795be2bde40de334fc8e26413a3d8a620a137be90d1f2c164e" +task.json| }, +task.json| "edge_mutation_script_versioned": true, +task.json| "desired_state_script_count_after_apply": 5, +task.json| "all_active_skladchik_ssh_requires_strict_known_hosts": true, +task.json| "unsafe_active_hostkey_markers_after_verify": 0, +task.json| "pve01_assets_checked_by_repository_tests": true, +task.json| "regression_locks": [ +task.json| "REG-024-SKLADCHIK-REAUTH-STRICT-HOSTKEY", +task.json| "REG-025-EXACT-SHA256-LENGTH-VALIDATION", +task.json| "REG-026-ACTIVE-SKLADCHIK-STRICT-HOSTKEY-COVERAGE", +task.json| "REG-027-REMOTE-MUTATION-SCRIPT-MUST-BE-VERSIONED", +task.json| "REG-028-FORBIDDEN-TOKEN-IN-NEGATIVE-ASSERTION", +task.json| "REG-029-LEGACY-WRAPPER-NOT-COPIED-VERBATIM", +task.json| "REG-030-BINARY-ARTIFACT-TEXT-SCAN", +task.json| "REG-031-SEMANTIC-TEST-LOOP-EXPANSION-MISMATCH", +task.json| "REG-032-OVERBROAD-INLINE-PYTHON-TEST", +task.json| "REG-033-INSTALLED-SOURCE-MUST-BE-DECLARED", +task.json| "REG-034-ROLLBACK-STAGE-PARENT-MODE", +RUN_EVIDENCE_END +CURRENT_FACTS_BEGIN +EDGE_SCRIPT_LOCAL_EXISTS=true +EDGE_SCRIPT_LOCAL_META=mode=700|uid=0|gid=0|size=5247 +EDGE_SCRIPT_LOCAL_SHA256=10fe854274dca1a053cfd8c0155340bd689bf3f6555c70dbc14b890cbafb93fa +LOCAL_HEALTH_EXISTS=false +SECRETS_ENV_EXISTS=false +MONITOR_PY_EXISTS=false +UNIT=skladchik-reports-monitor.service|active=inactive|enabled=not-found +UNIT=skladchik-reports-monitor.timer|active=inactive|enabled=not-found +UNIT=skladchik-reports-monitor-supervisor.service|active=failed|enabled=static +UNIT=skladchik-reports-monitor-supervisor.timer|active=active|enabled=enabled +CURRENT_FACTS_END +RCA_CAPTURE_COMPLETE=true +CHANGES_MADE_BY_943=false ROLLBACK_STARTED=false ROLLBACK_RESTORED=null -HOMELAB_RESULT_CONTRACT={"version":1,"command_id":"SUPPORT-260923-SKLADCHIK-COOKIE-REFRESH-REISSUE-942R1","status":"FAIL","changes_made":true,"rollback_started":false,"rollback_restored":null} -SKLADCHIK942_END=true +HOMELAB_RESULT_CONTRACT={"version":1,"command_id":"SUPPORT-260923-SKLADCHIK-PREFLIGHT-LINE13-RCA-943R1","status":"OK","changes_made":false,"rollback_started":false,"rollback_restored":null} +SKLADCHIK943_END=true OUTPUT_END CHAT_OUTPUT_END