runtime: publish SUPPORT-260923-SKLADCHIK-PREFLIGHT-LINE13-RCA-943R1
This commit is contained in:
homelab-runtime-publisher
File diff suppressed because one or more lines are too long
@@ -0,0 +1,349 @@
|
||||
CHAT_OUTPUT_BEGIN
|
||||
COMMAND_ID=SUPPORT-260923-SKLADCHIK-PREFLIGHT-LINE13-RCA-943R1
|
||||
STATUS=OK
|
||||
RC=0
|
||||
HOST=pve01
|
||||
MODE=read-only
|
||||
COMPONENT=skladchik-preflight-line13-rca
|
||||
REFERENCE_REGISTER_CHECK=OK
|
||||
REFERENCE_SHA256=5a3d8e5154c41cb582a4e0aca68090be1f0138918bf82131a948df326f9d8d66
|
||||
ERROR_REGISTER_CHECK=OK
|
||||
ERROR_REGISTER_SHA256=3b09a553ec0f527ed3afeed4753f52a74ee3036045bbfb3c685e9f8af4ba7ba0
|
||||
COMMAND_SHA256=dd3a88084befa0f31cc5f49ef676cb01eec2dfd507c1e1bfd993fba637e753d7
|
||||
DUPLICATE_FAILED_COMMAND_BLOCKED=false
|
||||
EXECUTION_STARTED=true
|
||||
CHANGE_DECLARED=false
|
||||
RESULT_CONTRACT_VALID=true
|
||||
RESULT_CONTRACT_STATUS=NOT_APPLICABLE
|
||||
RESULT_CONTRACT_ERROR=NONE
|
||||
COMMAND_RC=0
|
||||
CHANGES_MADE=false
|
||||
ROLLBACK_STARTED=false
|
||||
ROLLBACK_RESTORED=null
|
||||
MUTATION_OUTCOME=NO_MUTATION
|
||||
SANITIZED=yes
|
||||
SECRETS_INCLUDED=no
|
||||
PRIVATE_ADDRESSES_INCLUDED=no
|
||||
RAW_EVIDENCE_SHA256=2c42ff80da827c41e303d1369683a91c76fe254e8a5cc304eeb6038334a8dafa
|
||||
SANITIZED_OUTPUT_SHA256=17ace4d0e724d6b41d9460dad37dfe85e5b452f724ea763d0ba457d6ad255c77
|
||||
OUTPUT_BEGIN
|
||||
SKLADCHIK943_BEGIN=true
|
||||
COMMAND_ID=SUPPORT-260923-SKLADCHIK-PREFLIGHT-LINE13-RCA-943R1
|
||||
MODE=read-only
|
||||
COMPONENT=skladchik-preflight-line13-rca
|
||||
PREDECESSOR_942=FAILED_STAGE_PREFLIGHT;DETAIL_phase_preflight_line_13_rc_1
|
||||
MUTATION_BOUNDARY=NONE_READ_ONLY
|
||||
PHASE_PREFLIGHT_SOURCE_BEGIN
|
||||
PHASE_PREFLIGHT_SOURCE_PRESENT=true
|
||||
PHASE_PREFLIGHT_SOURCE_SHA256=f3e533ebd72ba2343a78797a8087c529c9fcfa2b3cabc45b3c0bd477356a8cc9
|
||||
PHASE_PREFLIGHT_SOURCE_BYTES=3076
|
||||
PHASE_PREFLIGHT_SOURCE_LINES=29
|
||||
0001:#!/usr/bin/env bash
|
||||
0002:set -Eeuo pipefail
|
||||
0003:on_err(){ local rc=$?; trap - ERR; printf 'ROOT_CAUSE=UNEXPECTED_PHASE_ERROR\nROOT_CAUSE_COUNT=1\nDETAIL=phase=preflight line=%s rc=%s\n' "${1:-0}" "$rc"; exit "$rc"; }
|
||||
0004:trap 'on_err "$LINENO"' ERR
|
||||
0005:export SYSTEMD_PAGER=cat PAGER=cat SYSTEMD_LESS=FRXMK
|
||||
0006:TASK_ID=skladchik-reports-monitor-controlled-reauth-v1
|
||||
0007:EDGE=debian@[PRIVATE_IP]
|
||||
0008:CURRENT_EDGE_SHA=b13d846cf089d450e3d5ac9e85ad1e232c198930e466bb8b82f6983a4d48ce0e
|
||||
0009:TARGET_EDGE_SHA=10fe854274dca1a053cfd8c0155340bd689bf3f6555c70dbc14b890cbafb93fa
|
||||
0010:SSH=(-o BatchMode=yes -o ConnectTimeout=12 -o StrictHostKeyChecking=yes -o UserKnownHostsFile=[SENSITIVE_PATH]
|
||||
0011:test "$(hostname -f)" = pve01.gram1.ru
|
||||
0012:test -z "$(git -C /srv/homelab-desired-state status --porcelain)"
|
||||
0013:test "$(sha256sum /usr/local/sbin/skladchik-reports-monitor-cookie-update | awk '{print $1}')" = a3006723d902fb7735c593003a6c6755fa3d69683126ec5f53165131f754fd7a
|
||||
0014:test "$(sha256sum /usr/local/sbin/skladchik-reports-monitor-backup | awk '{print $1}')" = ad0652d1e154fca94a20bdf831b40b831fd0d862eb104ec3377f486e96270d0a
|
||||
0015:test "$(sha256sum /usr/local/sbin/skladchik-reports-monitor-monthly-selftest-watch | awk '{print $1}')" = dd2c7e26aaa6835fa4ac4365eade338eb7b19f4bfe7528179c59cf8039454180
|
||||
0016:test "$(sha256sum /usr/local/sbin/skladchik-reports-monitor-status | awk '{print $1}')" = f94ec75155f415b35517dade0bfba00926eba6088e0c2b5c96ae79daa5dc368f
|
||||
0017:test "$(sha256sum /usr/local/sbin/skladchik-reports-monitor-supervisor | awk '{print $1}')" = 69494a4c32fa632844cdb54aabbe302a14d478924efcb719c6cd3d594d102369
|
||||
0018:test "$(sha256sum /srv/homelab-desired-state/sbin/skladchik-reports-monitor-backup | awk '{print $1}')" = ad0652d1e154fca94a20bdf831b40b831fd0d862eb104ec3377f486e96270d0a
|
||||
0019:test "$(sha256sum "$HOMELAB_TASK_DIR/assets/edge/skladchik-reports-monitor-cookie-update-edge" | awk '{print $1}')" = "$TARGET_EDGE_SHA"
|
||||
0020:RESULT="$(ssh "${SSH[@]}" "$EDGE" sudo -n /bin/bash -s -- "$CURRENT_EDGE_SHA" < "$HOMELAB_TASK_DIR/assets/edge-preflight.sh")"
|
||||
0021:grep -Fq 'EDGE_PREFLIGHT_STATUS=OK' <<<"$RESULT"
|
||||
0022:test "$(systemctl is-enabled skladchik-reports-monitor-supervisor.timer)" = enabled
|
||||
0023:test "$(systemctl is-active skladchik-reports-monitor-supervisor.timer)" = active
|
||||
0024:test "$(systemctl show skladchik-reports-monitor-supervisor.service -p Result --value)" = exit-code
|
||||
0025:grep -q '^STATUS=OK$' /var/lib/skladchik-reports-monitor-backup/health.txt
|
||||
0026:grep -q '^STATUS=OK$' /var/lib/skladchik-reports-monitor-backup/restore-check-health.txt
|
||||
0027:jq -n --arg status OK --arg current_edge_script_sha "$CURRENT_EDGE_SHA" --arg target_edge_script_sha "$TARGET_EDGE_SHA" '{status:$status,root_cause_count:0,production_changed:false,edge_monitor_status:"AUTH_REQUIRED",current_edge_script_sha256:$current_edge_script_sha,target_edge_script_sha256:$target_edge_script_sha,strict_hostkey:true,active_skladchik_sources_pinned:true}' > "$HOMELAB_RUN_DIR/preflight.json"
|
||||
0028:printf '%s\n' "$RESULT"
|
||||
0029:printf 'PREFLIGHT_STATUS=OK\nACTIVE_SKLADCHIK_SOURCE_HASHES=PINNED\nEDGE_MUTATION_SCRIPT=VERSIONED\nROOT_CAUSE_COUNT=0\nPRODUCTION_CHANGED=NO\nREAUTH_EXECUTED=NO\n'
|
||||
PHASE_PREFLIGHT_SOURCE_END
|
||||
EDGE_PREFLIGHT_SOURCE_BEGIN
|
||||
EDGE_PREFLIGHT_SOURCE_PRESENT=true
|
||||
EDGE_PREFLIGHT_SOURCE_SHA256=77ebb2893962550f97d80bdfd2eb414f666578a272b6c35a3f4ca669df117878
|
||||
EDGE_PREFLIGHT_SOURCE_BYTES=1351
|
||||
EDGE_PREFLIGHT_SOURCE_LINES=29
|
||||
0001:#!/usr/bin/env bash
|
||||
0002:set -Eeuo pipefail
|
||||
0003:EXPECTED_SHA="${1:?expected script sha required}"
|
||||
0004:SCRIPT=/usr/local/sbin/skladchik-reports-monitor-cookie-update-edge
|
||||
0005:HEALTH=/var/lib/skladchik-reports-monitor/health.txt
|
||||
0006:test -x "$SCRIPT"
|
||||
0007:test "$(sha256sum "$SCRIPT" | awk '{print $1}')" = "$EXPECTED_SHA"
|
||||
0008:test -s /etc/skladchik-reports-monitor/secrets.env
|
||||
0009:test -s /opt/skladchik-reports-monitor/monitor.py
|
||||
0010:test -s "$HEALTH"
|
||||
0011:STATUS="$(awk -F= '$1=="STATUS"{print $2;exit}' "$HEALTH")"
|
||||
0012:test "$STATUS" = AUTH_REQUIRED
|
||||
0013:test "$(systemctl is-enabled skladchik-reports-monitor.timer)" = enabled
|
||||
0014:test "$(systemctl is-active skladchik-reports-monitor.timer)" = active
|
||||
0015:WAIT_SECONDS=90
|
||||
0016:DEADLINE=$((SECONDS+WAIT_SECONDS))
|
||||
0017:monitor_busy() {
|
||||
0018: systemctl is-active --quiet skladchik-reports-monitor.service ||
|
||||
0019: pgrep -f '/opt/skladchik-reports-monitor/monitor.py' >/dev/null ||
|
||||
0020: pgrep -f 'skladchik-reports-monitor-cookie-update-edge' >/dev/null
|
||||
0021:}
|
||||
0022:while monitor_busy; do
|
||||
0023: if (( SECONDS >= DEADLINE )); then
|
||||
0024: printf 'EDGE_PREFLIGHT_STATUS=BUSY\nFAILED_PHASE=WAIT_MONITOR_QUIESCENCE\nWAIT_SECONDS=%s\n' "$WAIT_SECONDS"
|
||||
0025: exit 75
|
||||
0026: fi
|
||||
0027: sleep 2
|
||||
0028:done
|
||||
0029:printf 'EDGE_PREFLIGHT_STATUS=OK\nEDGE_MONITOR_STATUS=%s\nEDGE_SCRIPT_SHA256=%s\nEDGE_TIMER_ENABLED=YES\nEDGE_TIMER_ACTIVE=YES\nMONITOR_QUIESCENCE=OK\nWAIT_LIMIT_SECONDS=%s\n' "$STATUS" "$EXPECTED_SHA" "$WAIT_SECONDS"
|
||||
EDGE_PREFLIGHT_SOURCE_END
|
||||
FAILED_LINE_CONTEXT_BEGIN
|
||||
PHASE_LINE_0009=TARGET_EDGE_SHA=10fe854274dca1a053cfd8c0155340bd689bf3f6555c70dbc14b890cbafb93fa
|
||||
PHASE_LINE_0010=SSH=(-o BatchMode=yes -o ConnectTimeout=12 -o StrictHostKeyChecking=yes -o UserKnownHostsFile=[SENSITIVE_PATH]
|
||||
PHASE_LINE_0011=test "$(hostname -f)" = pve01.gram1.ru
|
||||
PHASE_LINE_0012=test -z "$(git -C /srv/homelab-desired-state status --porcelain)"
|
||||
PHASE_LINE_0013=test "$(sha256sum /usr/local/sbin/skladchik-reports-monitor-cookie-update | awk '{print $1}')" = a3006723d902fb7735c593003a6c6755fa3d69683126ec5f53165131f754fd7a
|
||||
PHASE_LINE_0014=test "$(sha256sum /usr/local/sbin/skladchik-reports-monitor-backup | awk '{print $1}')" = ad0652d1e154fca94a20bdf831b40b831fd0d862eb104ec3377f486e96270d0a
|
||||
PHASE_LINE_0015=test "$(sha256sum /usr/local/sbin/skladchik-reports-monitor-monthly-selftest-watch | awk '{print $1}')" = dd2c7e26aaa6835fa4ac4365eade338eb7b19f4bfe7528179c59cf8039454180
|
||||
PHASE_LINE_0016=test "$(sha256sum /usr/local/sbin/skladchik-reports-monitor-status | awk '{print $1}')" = f94ec75155f415b35517dade0bfba00926eba6088e0c2b5c96ae79daa5dc368f
|
||||
PHASE_LINE_0017=test "$(sha256sum /usr/local/sbin/skladchik-reports-monitor-supervisor | awk '{print $1}')" = 69494a4c32fa632844cdb54aabbe302a14d478924efcb719c6cd3d594d102369
|
||||
EDGE_LINE_0009=test -s /opt/skladchik-reports-monitor/monitor.py
|
||||
EDGE_LINE_0010=test -s "$HEALTH"
|
||||
EDGE_LINE_0011=STATUS="$(awk -F= '$1=="STATUS"{print $2;exit}' "$HEALTH")"
|
||||
EDGE_LINE_0012=test "$STATUS" = AUTH_REQUIRED
|
||||
EDGE_LINE_0013=test "$(systemctl is-enabled skladchik-reports-monitor.timer)" = enabled
|
||||
EDGE_LINE_0014=test "$(systemctl is-active skladchik-reports-monitor.timer)" = active
|
||||
EDGE_LINE_0015=WAIT_SECONDS=90
|
||||
EDGE_LINE_0016=DEADLINE=$((SECONDS+WAIT_SECONDS))
|
||||
EDGE_LINE_0017=monitor_busy() {
|
||||
FAILED_LINE_CONTEXT_END
|
||||
RUN_EVIDENCE_BEGIN
|
||||
RUN_FILE=stdout.log|present=true|bytes=89
|
||||
stdout.log|ROOT_CAUSE=UNEXPECTED_PHASE_ERROR
|
||||
stdout.log|ROOT_CAUSE_COUNT=1
|
||||
stdout.log|DETAIL=phase=preflight line=13 rc=1
|
||||
RUN_FILE=stderr.log|present=true|bytes=0
|
||||
RUN_FILE=preflight.json|present=false|bytes=0
|
||||
RUN_FILE=result.json|present=true|bytes=592
|
||||
result.json|{
|
||||
result.json| "run_id": "20260923T120316Z-986653-preflight",
|
||||
result.json| "task_id": "skladchik-reports-monitor-controlled-reauth-v1",
|
||||
result.json| "phase": "preflight",
|
||||
result.json| "mutation_scope": "controlled_apply",
|
||||
result.json| "production_change": true,
|
||||
result.json| "started_at": "2026-09-23T12:03:16Z",
|
||||
result.json| "finished_at": "2026-09-23T12:03:16Z",
|
||||
result.json| "status": "FAILED",
|
||||
result.json| "rc": 1,
|
||||
result.json| "stdout": "/var/lib/homelab-admin/runs/skladchik-reports-monitor-controlled-reauth-v1/20260923T120316Z-986653-preflight/stdout.log",
|
||||
result.json| "stderr": "/var/lib/homelab-admin/runs/skladchik-reports-monitor-controlled-reauth-v1/20260923T120316Z-986653-preflight/stderr.log"
|
||||
result.json|}
|
||||
RUN_FILE=task.json|present=true|bytes=13579
|
||||
task.json|{
|
||||
task.json| "schema_version": 2,
|
||||
task.json| "task_id": "skladchik-reports-monitor-controlled-reauth-v1",
|
||||
task.json| "target_host": "pve01.gram1.ru",
|
||||
task.json| "mutation_scope": "controlled_apply",
|
||||
task.json| "production_change": true,
|
||||
task.json| "failure_mode": "fail_fast",
|
||||
task.json| "guards": {
|
||||
task.json| "desired_state_clean": true,
|
||||
task.json| "failed_units_max": 4,
|
||||
task.json| "active_backup_restore_max": 0
|
||||
task.json| },
|
||||
task.json| "reads": [
|
||||
task.json| "[SENSITIVE_PATH]
|
||||
task.json| "/var/lib/skladchik-reports-monitor-backup/health.txt",
|
||||
task.json| "/var/lib/skladchik-reports-monitor-backup/restore-check-health.txt",
|
||||
task.json| "systemd:skladchik-reports-monitor-supervisor.service",
|
||||
task.json| "systemd:skladchik-reports-monitor-supervisor.timer",
|
||||
task.json| "ssh:debian@[PRIVATE_IP]:/usr/local/sbin/skladchik-reports-monitor-cookie-update-edge",
|
||||
task.json| "ssh:debian@[PRIVATE_IP]:/etc/skladchik-reports-monitor/secrets.env",
|
||||
task.json| "ssh:debian@[PRIVATE_IP]:/var/lib/skladchik-reports-monitor/cookies.jar",
|
||||
task.json| "ssh:debian@[PRIVATE_IP]:/var/lib/skladchik-reports-monitor/health.txt",
|
||||
task.json| "/usr/local/sbin/skladchik-reports-monitor-cookie-update",
|
||||
task.json| "/usr/local/sbin/skladchik-reports-monitor-backup",
|
||||
task.json| "/usr/local/sbin/skladchik-reports-monitor-monthly-selftest-watch",
|
||||
task.json| "/usr/local/sbin/skladchik-reports-monitor-status",
|
||||
task.json| "/usr/local/sbin/skladchik-reports-monitor-supervisor",
|
||||
task.json| "/srv/homelab-desired-state/sbin/skladchik-reports-monitor-backup",
|
||||
task.json| "/var/lib/skladchik-reports-monitor-supervisor"
|
||||
task.json| ],
|
||||
task.json| "writes": [
|
||||
task.json| "/var/lib/homelab-admin/seals/skladchik-reports-monitor-controlled-reauth-v1.json",
|
||||
task.json| "systemd:skladchik-reports-monitor-supervisor.service",
|
||||
task.json| "ssh:debian@[PRIVATE_IP]:/etc/skladchik-reports-monitor/secrets.env",
|
||||
task.json| "ssh:debian@[PRIVATE_IP]:/var/lib/skladchik-reports-monitor/cookies.jar",
|
||||
task.json| "ssh:debian@[PRIVATE_IP]:/var/lib/skladchik-reports-monitor/health.txt",
|
||||
task.json| "ssh:debian@[PRIVATE_IP]:/var/lib/homelab-admin/rollback/skladchik-reports-monitor-controlled-reauth-v1",
|
||||
task.json| "$HOMELAB_RUN_DIR/preflight.json",
|
||||
task.json| "$HOMELAB_RUN_DIR/prepare.json",
|
||||
task.json| "$HOMELAB_RUN_DIR/dry-run.json",
|
||||
task.json| "$HOMELAB_RUN_DIR/apply.json",
|
||||
task.json| "$HOMELAB_RUN_DIR/verify.json",
|
||||
task.json| "$HOMELAB_RUN_DIR/rollback.json",
|
||||
task.json| "$HOMELAB_RUN_DIR/seal.json",
|
||||
task.json| "/usr/local/sbin/skladchik-reports-monitor-cookie-update",
|
||||
task.json| "/usr/local/sbin/skladchik-reports-monitor-backup",
|
||||
task.json| "/usr/local/sbin/skladchik-reports-monitor-monthly-selftest-watch",
|
||||
task.json| "/usr/local/sbin/skladchik-reports-monitor-status",
|
||||
task.json| "/usr/local/sbin/skladchik-reports-monitor-supervisor",
|
||||
task.json| "/srv/homelab-desired-state/sbin/skladchik-reports-monitor-cookie-update",
|
||||
task.json| "/srv/homelab-desired-state/sbin/skladchik-reports-monitor-backup",
|
||||
task.json| "/srv/homelab-desired-state/sbin/skladchik-reports-monitor-monthly-selftest-watch",
|
||||
task.json| "/srv/homelab-desired-state/sbin/skladchik-reports-monitor-status",
|
||||
task.json| "/srv/homelab-desired-state/sbin/skladchik-reports-monitor-supervisor",
|
||||
task.json| "/mnt/staging/homelab-admin-rollback/skladchik-reports-monitor-controlled-reauth-v1",
|
||||
task.json| "ssh:debian@[PRIVATE_IP]:/usr/local/sbin/skladchik-reports-monitor-cookie-update-edge"
|
||||
task.json| ],
|
||||
task.json| "mutates": [
|
||||
task.json| "systemd:skladchik-reports-monitor-supervisor.service",
|
||||
task.json| "ssh:debian@[PRIVATE_IP]:/etc/skladchik-reports-monitor/secrets.env",
|
||||
task.json| "ssh:debian@[PRIVATE_IP]:/var/lib/skladchik-reports-monitor/cookies.jar",
|
||||
task.json| "ssh:debian@[PRIVATE_IP]:systemd:skladchik-reports-monitor.service",
|
||||
task.json| "ssh:debian@[PRIVATE_IP]:systemd:skladchik-reports-monitor.timer",
|
||||
task.json| "/usr/local/sbin/skladchik-reports-monitor-cookie-update",
|
||||
task.json| "/usr/local/sbin/skladchik-reports-monitor-backup",
|
||||
task.json| "/usr/local/sbin/skladchik-reports-monitor-monthly-selftest-watch",
|
||||
task.json| "/usr/local/sbin/skladchik-reports-monitor-status",
|
||||
task.json| "/usr/local/sbin/skladchik-reports-monitor-supervisor",
|
||||
task.json| "git:/srv/homelab-desired-state",
|
||||
task.json| "ssh:debian@[PRIVATE_IP]:/usr/local/sbin/skladchik-reports-monitor-cookie-update-edge"
|
||||
task.json| ],
|
||||
task.json| "expected_outputs": [
|
||||
task.json| {
|
||||
task.json| "path": "$HOMELAB_RUN_DIR/preflight.json",
|
||||
task.json| "producer": "phases/preflight.sh",
|
||||
task.json| "verification": "strict host-key access, the pinned live predecessor edge script, the versioned target edge script, AUTH_REQUIRED and bounded monitor quiescence are confirmed; evidence records both SHAs"
|
||||
task.json| },
|
||||
task.json| {
|
||||
task.json| "path": "$HOMELAB_RUN_DIR/prepare.json",
|
||||
task.json| "producer": "phases/prepare.sh",
|
||||
task.json| "verification": "root-only local and edge rollback snapshots have self-contained SHA256 manifests"
|
||||
task.json| },
|
||||
task.json| {
|
||||
task.json| "path": "$HOMELAB_RUN_DIR/dry-run.json",
|
||||
task.json| "producer": "phases/dry-run.sh",
|
||||
task.json| "verification": "rollback manifests validate, the pre-apply live edge script remains the pinned predecessor, the versioned deployment asset matches the pinned target SHA256, and evidence records both SHAs"
|
||||
task.json| },
|
||||
task.json| {
|
||||
task.json| "path": "$HOMELAB_RUN_DIR/apply.json",
|
||||
task.json| "producer": "phases/apply.sh",
|
||||
task.json| "verification": "apply replaces the pinned predecessor edge script with the pinned target script, quiesces the timer, performs one controlled reauthentication and restores or enables the timer"
|
||||
task.json| },
|
||||
task.json| {
|
||||
task.json| "path": "$HOMELAB_RUN_DIR/verify.json",
|
||||
task.json| "producer": "phases/verify.sh",
|
||||
task.json| "verification": "edge monitor, counts, timer, supervisor, backup and restore are OK"
|
||||
task.json| },
|
||||
task.json| {
|
||||
task.json| "path": "$HOMELAB_RUN_DIR/rollback.json",
|
||||
task.json| "producer": "phases/rollback.sh",
|
||||
task.json| "verification": "previous auth artifacts and timer state are restored"
|
||||
task.json| },
|
||||
task.json| {
|
||||
task.json| "path": "$HOMELAB_RUN_DIR/seal.json",
|
||||
task.json| "producer": "phases/seal.sh",
|
||||
task.json| "verification": "standard seal exists and rollback snapshot is removed"
|
||||
task.json| }
|
||||
task.json| ],
|
||||
task.json| "runtime_matrix": [
|
||||
task.json| {
|
||||
task.json| "host": "pve01.gram1.ru",
|
||||
task.json| "runtime": "bash,jq,git,systemd,ssh",
|
||||
task.json| "compile": "bash -n; jq -e"
|
||||
task.json| },
|
||||
task.json| {
|
||||
task.json| "host": "edge",
|
||||
task.json| "runtime": "bash,python3,systemd,sha256sum",
|
||||
task.json| "compile": "bash -n"
|
||||
task.json| }
|
||||
task.json| ],
|
||||
task.json| "rollback": {
|
||||
task.json| "command": "HOMELAB_CONTROLLED_APPLY_CONFIRM=skladchik-reports-monitor-controlled-reauth-v1 /opt/homelab-admin/bin/homelab-admin run TASK_DIR rollback",
|
||||
task.json| "verification": "self-contained local and edge manifests validate; prior auth artifacts, scripts, desired-state HEAD, supervisor state and timer state are restorable"
|
||||
task.json| },
|
||||
task.json| "no_change_assertions": [
|
||||
task.json| "vm160_backup_job",
|
||||
task.json| "cluster_backup_jobs",
|
||||
task.json| "unrelated_guests",
|
||||
task.json| "network",
|
||||
task.json| "dns",
|
||||
task.json| "known_hosts",
|
||||
task.json| "mail_cloud",
|
||||
task.json| "gitea",
|
||||
task.json| "backup_and_restore_artifacts"
|
||||
task.json| ],
|
||||
task.json| "source_of_truth": {
|
||||
task.json| "change_id": "CR-2026-0012",
|
||||
task.json| "incident": "Skladchik monitor AUTH_REQUIRED with missing complaint counts",
|
||||
task.json| "edge_script_sha256": "10fe854274dca1a053cfd8c0155340bd689bf3f6555c70dbc14b890cbafb93fa",
|
||||
task.json| "edge_script_path": "/usr/local/sbin/skladchik-reports-monitor-cookie-update-edge",
|
||||
task.json| "unsafe_legacy_wrapper_path": "/usr/local/sbin/skladchik-reports-monitor-cookie-update",
|
||||
task.json| "unsafe_legacy_wrapper_reason": "legacy host-key auto-acceptance",
|
||||
task.json| "strict_hostkey_required": true,
|
||||
task.json| "hidden_interactive_prompt_required": true,
|
||||
task.json| "cookie_content_must_not_enter_git_or_logs": true,
|
||||
task.json| "backup_status_before": "OK",
|
||||
task.json| "restore_status_before": "OK",
|
||||
task.json| "regression_lock": "REG-024-SKLADCHIK-REAUTH-STRICT-HOSTKEY",
|
||||
task.json| "active_pve01_script_count": 5,
|
||||
task.json| "active_pve01_original_sha256": {
|
||||
task.json| "skladchik-reports-monitor-cookie-update": "a3006723d902fb7735c593003a6c6755fa3d69683126ec5f53165131f754fd7a",
|
||||
task.json| "skladchik-reports-monitor-backup": "ad0652d1e154fca94a20bdf831b40b831fd0d862eb104ec3377f486e96270d0a",
|
||||
task.json| "skladchik-reports-monitor-monthly-selftest-watch": "dd2c7e26aaa6835fa4ac4365eade338eb7b19f4bfe7528179c59cf8039454180",
|
||||
task.json| "skladchik-reports-monitor-status": "f94ec75155f415b35517dade0bfba00926eba6088e0c2b5c96ae79daa5dc368f",
|
||||
task.json| "skladchik-reports-monitor-supervisor": "69494a4c32fa632844cdb54aabbe302a14d478924efcb719c6cd3d594d102369"
|
||||
task.json| },
|
||||
task.json| "active_pve01_deployed_sha256": {
|
||||
task.json| "skladchik-reports-monitor-cookie-update": "ea2d89ce67893a6d89b29610e4d46e207932ecc157173454585f67890f58810c",
|
||||
task.json| "skladchik-reports-monitor-backup": "1912f8f0559637563972bb8ac984fb0036694092056ae53968c02cc3b67855f3",
|
||||
task.json| "skladchik-reports-monitor-monthly-selftest-watch": "48c6ca01d2430d14c6af797c079640815ba1352a98466d4014ecbb80a27eaada",
|
||||
task.json| "skladchik-reports-monitor-status": "f902a61109efbac5918360f2235c2523bd10704ac484a4e8591e8a1c490faf5a",
|
||||
task.json| "skladchik-reports-monitor-supervisor": "35c68ae0929ff3795be2bde40de334fc8e26413a3d8a620a137be90d1f2c164e"
|
||||
task.json| },
|
||||
task.json| "edge_mutation_script_versioned": true,
|
||||
task.json| "desired_state_script_count_after_apply": 5,
|
||||
task.json| "all_active_skladchik_ssh_requires_strict_known_hosts": true,
|
||||
task.json| "unsafe_active_hostkey_markers_after_verify": 0,
|
||||
task.json| "pve01_assets_checked_by_repository_tests": true,
|
||||
task.json| "regression_locks": [
|
||||
task.json| "REG-024-SKLADCHIK-REAUTH-STRICT-HOSTKEY",
|
||||
task.json| "REG-025-EXACT-SHA256-LENGTH-VALIDATION",
|
||||
task.json| "REG-026-ACTIVE-SKLADCHIK-STRICT-HOSTKEY-COVERAGE",
|
||||
task.json| "REG-027-REMOTE-MUTATION-SCRIPT-MUST-BE-VERSIONED",
|
||||
task.json| "REG-028-FORBIDDEN-TOKEN-IN-NEGATIVE-ASSERTION",
|
||||
task.json| "REG-029-LEGACY-WRAPPER-NOT-COPIED-VERBATIM",
|
||||
task.json| "REG-030-BINARY-ARTIFACT-TEXT-SCAN",
|
||||
task.json| "REG-031-SEMANTIC-TEST-LOOP-EXPANSION-MISMATCH",
|
||||
task.json| "REG-032-OVERBROAD-INLINE-PYTHON-TEST",
|
||||
task.json| "REG-033-INSTALLED-SOURCE-MUST-BE-DECLARED",
|
||||
task.json| "REG-034-ROLLBACK-STAGE-PARENT-MODE",
|
||||
RUN_EVIDENCE_END
|
||||
CURRENT_FACTS_BEGIN
|
||||
EDGE_SCRIPT_LOCAL_EXISTS=true
|
||||
EDGE_SCRIPT_LOCAL_META=mode=700|uid=0|gid=0|size=5247
|
||||
EDGE_SCRIPT_LOCAL_SHA256=10fe854274dca1a053cfd8c0155340bd689bf3f6555c70dbc14b890cbafb93fa
|
||||
LOCAL_HEALTH_EXISTS=false
|
||||
SECRETS_ENV_EXISTS=false
|
||||
MONITOR_PY_EXISTS=false
|
||||
UNIT=skladchik-reports-monitor.service|active=inactive|enabled=not-found
|
||||
UNIT=skladchik-reports-monitor.timer|active=inactive|enabled=not-found
|
||||
UNIT=skladchik-reports-monitor-supervisor.service|active=failed|enabled=static
|
||||
UNIT=skladchik-reports-monitor-supervisor.timer|active=active|enabled=enabled
|
||||
CURRENT_FACTS_END
|
||||
RCA_CAPTURE_COMPLETE=true
|
||||
CHANGES_MADE_BY_943=false
|
||||
ROLLBACK_STARTED=false
|
||||
ROLLBACK_RESTORED=null
|
||||
HOMELAB_RESULT_CONTRACT={"version":1,"command_id":"SUPPORT-260923-SKLADCHIK-PREFLIGHT-LINE13-RCA-943R1","status":"OK","changes_made":false,"rollback_started":false,"rollback_restored":null}
|
||||
SKLADCHIK943_END=true
|
||||
|
||||
OUTPUT_END
|
||||
CHAT_OUTPUT_END
|
||||
+16
-16
File diff suppressed because one or more lines are too long
+328
-100
@@ -1,121 +1,349 @@
|
||||
CHAT_OUTPUT_BEGIN
|
||||
COMMAND_ID=SUPPORT-260923-SKLADCHIK-COOKIE-REFRESH-REISSUE-942R1
|
||||
STATUS=FAIL
|
||||
RC=1
|
||||
COMMAND_ID=SUPPORT-260923-SKLADCHIK-PREFLIGHT-LINE13-RCA-943R1
|
||||
STATUS=OK
|
||||
RC=0
|
||||
HOST=pve01
|
||||
MODE=change
|
||||
COMPONENT=skladchik-cookie-refresh-reissue
|
||||
MODE=read-only
|
||||
COMPONENT=skladchik-preflight-line13-rca
|
||||
REFERENCE_REGISTER_CHECK=OK
|
||||
REFERENCE_SHA256=5a3d8e5154c41cb582a4e0aca68090be1f0138918bf82131a948df326f9d8d66
|
||||
ERROR_REGISTER_CHECK=OK
|
||||
ERROR_REGISTER_SHA256=3b09a553ec0f527ed3afeed4753f52a74ee3036045bbfb3c685e9f8af4ba7ba0
|
||||
COMMAND_SHA256=27da7524dbfc342b7925b1034305a552a30bcb635ca4e09e5ee7c3bfdb3e51b6
|
||||
COMMAND_SHA256=dd3a88084befa0f31cc5f49ef676cb01eec2dfd507c1e1bfd993fba637e753d7
|
||||
DUPLICATE_FAILED_COMMAND_BLOCKED=false
|
||||
EXECUTION_STARTED=true
|
||||
CHANGE_DECLARED=true
|
||||
CHANGE_DECLARED=false
|
||||
RESULT_CONTRACT_VALID=true
|
||||
RESULT_CONTRACT_STATUS=FAIL
|
||||
RESULT_CONTRACT_STATUS=NOT_APPLICABLE
|
||||
RESULT_CONTRACT_ERROR=NONE
|
||||
COMMAND_RC=1
|
||||
CHANGES_MADE=true
|
||||
COMMAND_RC=0
|
||||
CHANGES_MADE=false
|
||||
ROLLBACK_STARTED=false
|
||||
ROLLBACK_RESTORED=null
|
||||
MUTATION_OUTCOME=MUTATED
|
||||
MUTATION_OUTCOME=NO_MUTATION
|
||||
SANITIZED=yes
|
||||
SECRETS_INCLUDED=no
|
||||
PRIVATE_ADDRESSES_INCLUDED=no
|
||||
RAW_EVIDENCE_SHA256=edad064e291af1b094dd2a4d47ae4372a5fa3ddfdb4518232751e42562c271f5
|
||||
SANITIZED_OUTPUT_SHA256=a51b6642d8cde7b817b5bfd3ee7d5b5a3b22459a9dee1f3d368a3ef292aebb3e
|
||||
RAW_EVIDENCE_SHA256=2c42ff80da827c41e303d1369683a91c76fe254e8a5cc304eeb6038334a8dafa
|
||||
SANITIZED_OUTPUT_SHA256=17ace4d0e724d6b41d9460dad37dfe85e5b452f724ea763d0ba457d6ad255c77
|
||||
OUTPUT_BEGIN
|
||||
SKLADCHIK942_BEGIN=true
|
||||
COMMAND_ID=SUPPORT-260923-SKLADCHIK-COOKIE-REFRESH-REISSUE-942R1
|
||||
MODE=change
|
||||
COMPONENT=skladchik-cookie-refresh-reissue
|
||||
PREDECESSOR_929=PASS_SKLADCHIK_929_TRANSPORT_RECONCILE_SAFE_TO_REISSUE
|
||||
PREDECESSOR_931=LOCAL_TRANSPORT_RC126_NO_RUNNER_RESULT
|
||||
PREDECESSOR_932=PASS_SKLADCHIK_931_LOCAL_CARRIER_FORENSICS
|
||||
PREDECESSOR_933=HOLD_SKLADCHIK_933_COOKIE_STDIN_INVALID_NO_MUTATION
|
||||
PREDECESSOR_934=HOLD_SKLADCHIK_934_COOKIE_STDIN_INVALID_NO_MUTATION
|
||||
931_RCA=UPLOAD_SUCCEEDED;DIRECT_EXEC_FROM_RUN_RETURNED_126;STDERR_63_BYTES_MATCHES_BASH_PERMISSION_DENIED;FIX_RUN_OUTER_VIA_USR_BIN_BASH
|
||||
933_RCA=RUNNER_REACHED;COOKIE_CLIPBOARD_STDIN_INVALID;NO_MUTATION
|
||||
934_RCA=RUNNER_REACHED;READ_SUCCEEDED_BUT_COOKIE_VALIDATION_FAILED;NO_MUTATION;COMMON_FAILURE_IS_WINDOWS_INPUT_TRANSPORT
|
||||
935_TRANSPORT_PROOF=MASKED_GUI_TEXTBOX_WITH_PASTE;DIRECT_WRITE_TO_PLINK_PROCESS_STANDARDINPUT;COOKIE_INPUT_LENGTH_325_REACHED_RUNNER
|
||||
PREDECESSOR_935=OFFICIAL_REFRESH_REACHED_INITIAL_STATE_CHECK;FAILED_BEFORE_COOKIE_PROMPT
|
||||
PREDECESSOR_939=ONLY_INITIAL_CHECK_LINE_159_FAILED
|
||||
PREDECESSOR_940=LOCAL_BEHIND_FAST_FORWARD_TO_ORIGIN
|
||||
PREDECESSOR_941=PASS_HOMELAB_OPS_FASTFORWARD;INITIAL_STATE_CHECK_ALL_PASS
|
||||
942_PURPOSE=REISSUE_OFFICIAL_COOKIE_REFRESH_AFTER_LINE_159_REMEDIATION
|
||||
MUTATION_BOUNDARY=OFFICIAL_SKLADCHIK_COOKIE_REFRESH_ONLY;TRANSIENT_ROOT_ONLY_COOKIE_STAGING;NO_OTHER_SERVICES;NO_VM_CONFIG;NO_CLOUD_CONFIG
|
||||
COOKIE_INPUT_LENGTH=325
|
||||
INITIAL_STATE_GIT_GATE=PASS
|
||||
RUNNER_FIRST_COOKIE_INPUT=true
|
||||
OFFICIAL_RUNBOOK_VERIFIED=true
|
||||
COOKIE_SOURCE=WINDOWS_CHROME_MASKED_GUI_DIRECT_PROCESS_STANDARDINPUT
|
||||
COOKIE_SECRET_OUTPUT=NONE
|
||||
SCOPE=SKLADCHIK_COOKIE_REFRESH_VALIDATE_PREFLIGHT_PREPARE_DRY_RUN_HIDDEN_COOKIE_APPLY_VERIFY_SEAL
|
||||
ROLLBACK=AUTOMATIC_VERSIONED_RECOVERY_WORKFLOW_RESTORES_PREVIOUS_SECRET_COOKIE_JAR_SCRIPTS_DESIRED_STATE_TIMER_AND_SUPERVISOR_STATE_ON_FAILURE_BEFORE_SEAL
|
||||
VERIFY=FRESH_COOKIE_FORCED_STALE_JAR_BYPASSED_SECOND_JAR_ONLY_PROBE_MONITOR_OK_NUMERIC_COUNTS_TARGET_EDGE_SHA_TIMERS_SUPERVISOR_SEAL_STAGE_REMOVAL_VM160_UNCHANGED_NO_CLOUD_COMMANDS
|
||||
LINT_STATUS=OK
|
||||
TASK_ID=skladchik-reports-monitor-controlled-reauth-v1
|
||||
POLICY_ID=homelab-admin-guardrails-v1.2
|
||||
RULES_CHECKED=14
|
||||
SOURCE_FILES_CHECKED=19
|
||||
TASK_VALID=YES
|
||||
TASK_ID=skladchik-reports-monitor-controlled-reauth-v1
|
||||
TASK_SCOPE=controlled_apply
|
||||
PRODUCTION_CHANGE=YES
|
||||
LINT_STATUS=OK
|
||||
TASK_ID=skladchik-reports-monitor-controlled-reauth-v1
|
||||
POLICY_ID=homelab-admin-guardrails-v1.2
|
||||
RULES_CHECKED=14
|
||||
SOURCE_FILES_CHECKED=19
|
||||
ROOT_CAUSE=UNEXPECTED_PHASE_ERROR
|
||||
ROOT_CAUSE_COUNT=1
|
||||
DETAIL=phase=preflight line=13 rc=1
|
||||
RUN_DIR=/var/lib/homelab-admin/runs/skladchik-reports-monitor-controlled-reauth-v1/20260923T120316Z-986653-preflight
|
||||
RUN_RC=1
|
||||
[1;35m===== СКОПИРОВАТЬ МНЕ В ЧАТ =====
|
||||
COOKIE_REFRESH_RESULT=FAILED
|
||||
FAILED_STAGE=PREFLIGHT
|
||||
FAILED_RC=1
|
||||
ROLLBACK_ATTEMPTED=NO
|
||||
ROLLBACK_RC=NOT_RUN
|
||||
SEALED=1
|
||||
BACKUP_WAIT_EVENTS=0
|
||||
ACTIVE_BACKUP_RESTORE_NOW=0
|
||||
APPLY_RUN=unknown
|
||||
COOKIE_VALUES_PRINTED=NO
|
||||
VM160_CHANGED=NO
|
||||
CLOUD_COMMANDS_EXECUTED=NO
|
||||
NEXT_PHASE=PASTE_THIS_FRAGMENT_ONLY
|
||||
===== КОНЕЦ ФРАГМЕНТА =====[0m
|
||||
PTY_COOKIE_FEED_SENT=false
|
||||
PTY_COOKIE_PROMPT_SEEN=false
|
||||
OFFICIAL_REFRESH_RC=1
|
||||
PTY_COOKIE_FEED_SENT=false
|
||||
PTY_COOKIE_PROMPT_SEEN=false
|
||||
COOKIE_REFRESH_RESULT_SUCCESS=false
|
||||
FRESH_COOKIE_FORCED=false
|
||||
STALE_COOKIE_JAR_BYPASSED=false
|
||||
POST_SUPERVISOR_STATUS=FAIL
|
||||
POST_MONITOR_STATUS=ERROR
|
||||
POST_MONITOR_REPORTS=missing
|
||||
POST_SESSION_KEEPALIVE=persistent_cookie_jar
|
||||
POST_COOKIE_EXPIRY_STATUS=WARN
|
||||
POST_COOKIE_JAR_PRESENT=false
|
||||
POST_COOKIE_JAR_BYTES=0
|
||||
POST_NETBIRD_ROUTE_WT0=false
|
||||
POST_ROUTE_PROBLEM_REPORTED=true
|
||||
FAILURE=OFFICIAL_REFRESH_RC_1
|
||||
COOKIE_REFRESH_COMPLETE=false
|
||||
DECISION=HOLD_SKLADCHIK_COOKIE_REFRESH_RCA
|
||||
NEXT_GATE=SKLADCHIK_COOKIE_REFRESH_RCA
|
||||
TASK_RESULT=HOLD_SKLADCHIK_COOKIE_REFRESH
|
||||
CHANGES_MADE_BY_942=true
|
||||
SKLADCHIK943_BEGIN=true
|
||||
COMMAND_ID=SUPPORT-260923-SKLADCHIK-PREFLIGHT-LINE13-RCA-943R1
|
||||
MODE=read-only
|
||||
COMPONENT=skladchik-preflight-line13-rca
|
||||
PREDECESSOR_942=FAILED_STAGE_PREFLIGHT;DETAIL_phase_preflight_line_13_rc_1
|
||||
MUTATION_BOUNDARY=NONE_READ_ONLY
|
||||
PHASE_PREFLIGHT_SOURCE_BEGIN
|
||||
PHASE_PREFLIGHT_SOURCE_PRESENT=true
|
||||
PHASE_PREFLIGHT_SOURCE_SHA256=f3e533ebd72ba2343a78797a8087c529c9fcfa2b3cabc45b3c0bd477356a8cc9
|
||||
PHASE_PREFLIGHT_SOURCE_BYTES=3076
|
||||
PHASE_PREFLIGHT_SOURCE_LINES=29
|
||||
0001:#!/usr/bin/env bash
|
||||
0002:set -Eeuo pipefail
|
||||
0003:on_err(){ local rc=$?; trap - ERR; printf 'ROOT_CAUSE=UNEXPECTED_PHASE_ERROR\nROOT_CAUSE_COUNT=1\nDETAIL=phase=preflight line=%s rc=%s\n' "${1:-0}" "$rc"; exit "$rc"; }
|
||||
0004:trap 'on_err "$LINENO"' ERR
|
||||
0005:export SYSTEMD_PAGER=cat PAGER=cat SYSTEMD_LESS=FRXMK
|
||||
0006:TASK_ID=skladchik-reports-monitor-controlled-reauth-v1
|
||||
0007:EDGE=debian@[PRIVATE_IP]
|
||||
0008:CURRENT_EDGE_SHA=b13d846cf089d450e3d5ac9e85ad1e232c198930e466bb8b82f6983a4d48ce0e
|
||||
0009:TARGET_EDGE_SHA=10fe854274dca1a053cfd8c0155340bd689bf3f6555c70dbc14b890cbafb93fa
|
||||
0010:SSH=(-o BatchMode=yes -o ConnectTimeout=12 -o StrictHostKeyChecking=yes -o UserKnownHostsFile=[SENSITIVE_PATH]
|
||||
0011:test "$(hostname -f)" = pve01.gram1.ru
|
||||
0012:test -z "$(git -C /srv/homelab-desired-state status --porcelain)"
|
||||
0013:test "$(sha256sum /usr/local/sbin/skladchik-reports-monitor-cookie-update | awk '{print $1}')" = a3006723d902fb7735c593003a6c6755fa3d69683126ec5f53165131f754fd7a
|
||||
0014:test "$(sha256sum /usr/local/sbin/skladchik-reports-monitor-backup | awk '{print $1}')" = ad0652d1e154fca94a20bdf831b40b831fd0d862eb104ec3377f486e96270d0a
|
||||
0015:test "$(sha256sum /usr/local/sbin/skladchik-reports-monitor-monthly-selftest-watch | awk '{print $1}')" = dd2c7e26aaa6835fa4ac4365eade338eb7b19f4bfe7528179c59cf8039454180
|
||||
0016:test "$(sha256sum /usr/local/sbin/skladchik-reports-monitor-status | awk '{print $1}')" = f94ec75155f415b35517dade0bfba00926eba6088e0c2b5c96ae79daa5dc368f
|
||||
0017:test "$(sha256sum /usr/local/sbin/skladchik-reports-monitor-supervisor | awk '{print $1}')" = 69494a4c32fa632844cdb54aabbe302a14d478924efcb719c6cd3d594d102369
|
||||
0018:test "$(sha256sum /srv/homelab-desired-state/sbin/skladchik-reports-monitor-backup | awk '{print $1}')" = ad0652d1e154fca94a20bdf831b40b831fd0d862eb104ec3377f486e96270d0a
|
||||
0019:test "$(sha256sum "$HOMELAB_TASK_DIR/assets/edge/skladchik-reports-monitor-cookie-update-edge" | awk '{print $1}')" = "$TARGET_EDGE_SHA"
|
||||
0020:RESULT="$(ssh "${SSH[@]}" "$EDGE" sudo -n /bin/bash -s -- "$CURRENT_EDGE_SHA" < "$HOMELAB_TASK_DIR/assets/edge-preflight.sh")"
|
||||
0021:grep -Fq 'EDGE_PREFLIGHT_STATUS=OK' <<<"$RESULT"
|
||||
0022:test "$(systemctl is-enabled skladchik-reports-monitor-supervisor.timer)" = enabled
|
||||
0023:test "$(systemctl is-active skladchik-reports-monitor-supervisor.timer)" = active
|
||||
0024:test "$(systemctl show skladchik-reports-monitor-supervisor.service -p Result --value)" = exit-code
|
||||
0025:grep -q '^STATUS=OK$' /var/lib/skladchik-reports-monitor-backup/health.txt
|
||||
0026:grep -q '^STATUS=OK$' /var/lib/skladchik-reports-monitor-backup/restore-check-health.txt
|
||||
0027:jq -n --arg status OK --arg current_edge_script_sha "$CURRENT_EDGE_SHA" --arg target_edge_script_sha "$TARGET_EDGE_SHA" '{status:$status,root_cause_count:0,production_changed:false,edge_monitor_status:"AUTH_REQUIRED",current_edge_script_sha256:$current_edge_script_sha,target_edge_script_sha256:$target_edge_script_sha,strict_hostkey:true,active_skladchik_sources_pinned:true}' > "$HOMELAB_RUN_DIR/preflight.json"
|
||||
0028:printf '%s\n' "$RESULT"
|
||||
0029:printf 'PREFLIGHT_STATUS=OK\nACTIVE_SKLADCHIK_SOURCE_HASHES=PINNED\nEDGE_MUTATION_SCRIPT=VERSIONED\nROOT_CAUSE_COUNT=0\nPRODUCTION_CHANGED=NO\nREAUTH_EXECUTED=NO\n'
|
||||
PHASE_PREFLIGHT_SOURCE_END
|
||||
EDGE_PREFLIGHT_SOURCE_BEGIN
|
||||
EDGE_PREFLIGHT_SOURCE_PRESENT=true
|
||||
EDGE_PREFLIGHT_SOURCE_SHA256=77ebb2893962550f97d80bdfd2eb414f666578a272b6c35a3f4ca669df117878
|
||||
EDGE_PREFLIGHT_SOURCE_BYTES=1351
|
||||
EDGE_PREFLIGHT_SOURCE_LINES=29
|
||||
0001:#!/usr/bin/env bash
|
||||
0002:set -Eeuo pipefail
|
||||
0003:EXPECTED_SHA="${1:?expected script sha required}"
|
||||
0004:SCRIPT=/usr/local/sbin/skladchik-reports-monitor-cookie-update-edge
|
||||
0005:HEALTH=/var/lib/skladchik-reports-monitor/health.txt
|
||||
0006:test -x "$SCRIPT"
|
||||
0007:test "$(sha256sum "$SCRIPT" | awk '{print $1}')" = "$EXPECTED_SHA"
|
||||
0008:test -s /etc/skladchik-reports-monitor/secrets.env
|
||||
0009:test -s /opt/skladchik-reports-monitor/monitor.py
|
||||
0010:test -s "$HEALTH"
|
||||
0011:STATUS="$(awk -F= '$1=="STATUS"{print $2;exit}' "$HEALTH")"
|
||||
0012:test "$STATUS" = AUTH_REQUIRED
|
||||
0013:test "$(systemctl is-enabled skladchik-reports-monitor.timer)" = enabled
|
||||
0014:test "$(systemctl is-active skladchik-reports-monitor.timer)" = active
|
||||
0015:WAIT_SECONDS=90
|
||||
0016:DEADLINE=$((SECONDS+WAIT_SECONDS))
|
||||
0017:monitor_busy() {
|
||||
0018: systemctl is-active --quiet skladchik-reports-monitor.service ||
|
||||
0019: pgrep -f '/opt/skladchik-reports-monitor/monitor.py' >/dev/null ||
|
||||
0020: pgrep -f 'skladchik-reports-monitor-cookie-update-edge' >/dev/null
|
||||
0021:}
|
||||
0022:while monitor_busy; do
|
||||
0023: if (( SECONDS >= DEADLINE )); then
|
||||
0024: printf 'EDGE_PREFLIGHT_STATUS=BUSY\nFAILED_PHASE=WAIT_MONITOR_QUIESCENCE\nWAIT_SECONDS=%s\n' "$WAIT_SECONDS"
|
||||
0025: exit 75
|
||||
0026: fi
|
||||
0027: sleep 2
|
||||
0028:done
|
||||
0029:printf 'EDGE_PREFLIGHT_STATUS=OK\nEDGE_MONITOR_STATUS=%s\nEDGE_SCRIPT_SHA256=%s\nEDGE_TIMER_ENABLED=YES\nEDGE_TIMER_ACTIVE=YES\nMONITOR_QUIESCENCE=OK\nWAIT_LIMIT_SECONDS=%s\n' "$STATUS" "$EXPECTED_SHA" "$WAIT_SECONDS"
|
||||
EDGE_PREFLIGHT_SOURCE_END
|
||||
FAILED_LINE_CONTEXT_BEGIN
|
||||
PHASE_LINE_0009=TARGET_EDGE_SHA=10fe854274dca1a053cfd8c0155340bd689bf3f6555c70dbc14b890cbafb93fa
|
||||
PHASE_LINE_0010=SSH=(-o BatchMode=yes -o ConnectTimeout=12 -o StrictHostKeyChecking=yes -o UserKnownHostsFile=[SENSITIVE_PATH]
|
||||
PHASE_LINE_0011=test "$(hostname -f)" = pve01.gram1.ru
|
||||
PHASE_LINE_0012=test -z "$(git -C /srv/homelab-desired-state status --porcelain)"
|
||||
PHASE_LINE_0013=test "$(sha256sum /usr/local/sbin/skladchik-reports-monitor-cookie-update | awk '{print $1}')" = a3006723d902fb7735c593003a6c6755fa3d69683126ec5f53165131f754fd7a
|
||||
PHASE_LINE_0014=test "$(sha256sum /usr/local/sbin/skladchik-reports-monitor-backup | awk '{print $1}')" = ad0652d1e154fca94a20bdf831b40b831fd0d862eb104ec3377f486e96270d0a
|
||||
PHASE_LINE_0015=test "$(sha256sum /usr/local/sbin/skladchik-reports-monitor-monthly-selftest-watch | awk '{print $1}')" = dd2c7e26aaa6835fa4ac4365eade338eb7b19f4bfe7528179c59cf8039454180
|
||||
PHASE_LINE_0016=test "$(sha256sum /usr/local/sbin/skladchik-reports-monitor-status | awk '{print $1}')" = f94ec75155f415b35517dade0bfba00926eba6088e0c2b5c96ae79daa5dc368f
|
||||
PHASE_LINE_0017=test "$(sha256sum /usr/local/sbin/skladchik-reports-monitor-supervisor | awk '{print $1}')" = 69494a4c32fa632844cdb54aabbe302a14d478924efcb719c6cd3d594d102369
|
||||
EDGE_LINE_0009=test -s /opt/skladchik-reports-monitor/monitor.py
|
||||
EDGE_LINE_0010=test -s "$HEALTH"
|
||||
EDGE_LINE_0011=STATUS="$(awk -F= '$1=="STATUS"{print $2;exit}' "$HEALTH")"
|
||||
EDGE_LINE_0012=test "$STATUS" = AUTH_REQUIRED
|
||||
EDGE_LINE_0013=test "$(systemctl is-enabled skladchik-reports-monitor.timer)" = enabled
|
||||
EDGE_LINE_0014=test "$(systemctl is-active skladchik-reports-monitor.timer)" = active
|
||||
EDGE_LINE_0015=WAIT_SECONDS=90
|
||||
EDGE_LINE_0016=DEADLINE=$((SECONDS+WAIT_SECONDS))
|
||||
EDGE_LINE_0017=monitor_busy() {
|
||||
FAILED_LINE_CONTEXT_END
|
||||
RUN_EVIDENCE_BEGIN
|
||||
RUN_FILE=stdout.log|present=true|bytes=89
|
||||
stdout.log|ROOT_CAUSE=UNEXPECTED_PHASE_ERROR
|
||||
stdout.log|ROOT_CAUSE_COUNT=1
|
||||
stdout.log|DETAIL=phase=preflight line=13 rc=1
|
||||
RUN_FILE=stderr.log|present=true|bytes=0
|
||||
RUN_FILE=preflight.json|present=false|bytes=0
|
||||
RUN_FILE=result.json|present=true|bytes=592
|
||||
result.json|{
|
||||
result.json| "run_id": "20260923T120316Z-986653-preflight",
|
||||
result.json| "task_id": "skladchik-reports-monitor-controlled-reauth-v1",
|
||||
result.json| "phase": "preflight",
|
||||
result.json| "mutation_scope": "controlled_apply",
|
||||
result.json| "production_change": true,
|
||||
result.json| "started_at": "2026-09-23T12:03:16Z",
|
||||
result.json| "finished_at": "2026-09-23T12:03:16Z",
|
||||
result.json| "status": "FAILED",
|
||||
result.json| "rc": 1,
|
||||
result.json| "stdout": "/var/lib/homelab-admin/runs/skladchik-reports-monitor-controlled-reauth-v1/20260923T120316Z-986653-preflight/stdout.log",
|
||||
result.json| "stderr": "/var/lib/homelab-admin/runs/skladchik-reports-monitor-controlled-reauth-v1/20260923T120316Z-986653-preflight/stderr.log"
|
||||
result.json|}
|
||||
RUN_FILE=task.json|present=true|bytes=13579
|
||||
task.json|{
|
||||
task.json| "schema_version": 2,
|
||||
task.json| "task_id": "skladchik-reports-monitor-controlled-reauth-v1",
|
||||
task.json| "target_host": "pve01.gram1.ru",
|
||||
task.json| "mutation_scope": "controlled_apply",
|
||||
task.json| "production_change": true,
|
||||
task.json| "failure_mode": "fail_fast",
|
||||
task.json| "guards": {
|
||||
task.json| "desired_state_clean": true,
|
||||
task.json| "failed_units_max": 4,
|
||||
task.json| "active_backup_restore_max": 0
|
||||
task.json| },
|
||||
task.json| "reads": [
|
||||
task.json| "[SENSITIVE_PATH]
|
||||
task.json| "/var/lib/skladchik-reports-monitor-backup/health.txt",
|
||||
task.json| "/var/lib/skladchik-reports-monitor-backup/restore-check-health.txt",
|
||||
task.json| "systemd:skladchik-reports-monitor-supervisor.service",
|
||||
task.json| "systemd:skladchik-reports-monitor-supervisor.timer",
|
||||
task.json| "ssh:debian@[PRIVATE_IP]:/usr/local/sbin/skladchik-reports-monitor-cookie-update-edge",
|
||||
task.json| "ssh:debian@[PRIVATE_IP]:/etc/skladchik-reports-monitor/secrets.env",
|
||||
task.json| "ssh:debian@[PRIVATE_IP]:/var/lib/skladchik-reports-monitor/cookies.jar",
|
||||
task.json| "ssh:debian@[PRIVATE_IP]:/var/lib/skladchik-reports-monitor/health.txt",
|
||||
task.json| "/usr/local/sbin/skladchik-reports-monitor-cookie-update",
|
||||
task.json| "/usr/local/sbin/skladchik-reports-monitor-backup",
|
||||
task.json| "/usr/local/sbin/skladchik-reports-monitor-monthly-selftest-watch",
|
||||
task.json| "/usr/local/sbin/skladchik-reports-monitor-status",
|
||||
task.json| "/usr/local/sbin/skladchik-reports-monitor-supervisor",
|
||||
task.json| "/srv/homelab-desired-state/sbin/skladchik-reports-monitor-backup",
|
||||
task.json| "/var/lib/skladchik-reports-monitor-supervisor"
|
||||
task.json| ],
|
||||
task.json| "writes": [
|
||||
task.json| "/var/lib/homelab-admin/seals/skladchik-reports-monitor-controlled-reauth-v1.json",
|
||||
task.json| "systemd:skladchik-reports-monitor-supervisor.service",
|
||||
task.json| "ssh:debian@[PRIVATE_IP]:/etc/skladchik-reports-monitor/secrets.env",
|
||||
task.json| "ssh:debian@[PRIVATE_IP]:/var/lib/skladchik-reports-monitor/cookies.jar",
|
||||
task.json| "ssh:debian@[PRIVATE_IP]:/var/lib/skladchik-reports-monitor/health.txt",
|
||||
task.json| "ssh:debian@[PRIVATE_IP]:/var/lib/homelab-admin/rollback/skladchik-reports-monitor-controlled-reauth-v1",
|
||||
task.json| "$HOMELAB_RUN_DIR/preflight.json",
|
||||
task.json| "$HOMELAB_RUN_DIR/prepare.json",
|
||||
task.json| "$HOMELAB_RUN_DIR/dry-run.json",
|
||||
task.json| "$HOMELAB_RUN_DIR/apply.json",
|
||||
task.json| "$HOMELAB_RUN_DIR/verify.json",
|
||||
task.json| "$HOMELAB_RUN_DIR/rollback.json",
|
||||
task.json| "$HOMELAB_RUN_DIR/seal.json",
|
||||
task.json| "/usr/local/sbin/skladchik-reports-monitor-cookie-update",
|
||||
task.json| "/usr/local/sbin/skladchik-reports-monitor-backup",
|
||||
task.json| "/usr/local/sbin/skladchik-reports-monitor-monthly-selftest-watch",
|
||||
task.json| "/usr/local/sbin/skladchik-reports-monitor-status",
|
||||
task.json| "/usr/local/sbin/skladchik-reports-monitor-supervisor",
|
||||
task.json| "/srv/homelab-desired-state/sbin/skladchik-reports-monitor-cookie-update",
|
||||
task.json| "/srv/homelab-desired-state/sbin/skladchik-reports-monitor-backup",
|
||||
task.json| "/srv/homelab-desired-state/sbin/skladchik-reports-monitor-monthly-selftest-watch",
|
||||
task.json| "/srv/homelab-desired-state/sbin/skladchik-reports-monitor-status",
|
||||
task.json| "/srv/homelab-desired-state/sbin/skladchik-reports-monitor-supervisor",
|
||||
task.json| "/mnt/staging/homelab-admin-rollback/skladchik-reports-monitor-controlled-reauth-v1",
|
||||
task.json| "ssh:debian@[PRIVATE_IP]:/usr/local/sbin/skladchik-reports-monitor-cookie-update-edge"
|
||||
task.json| ],
|
||||
task.json| "mutates": [
|
||||
task.json| "systemd:skladchik-reports-monitor-supervisor.service",
|
||||
task.json| "ssh:debian@[PRIVATE_IP]:/etc/skladchik-reports-monitor/secrets.env",
|
||||
task.json| "ssh:debian@[PRIVATE_IP]:/var/lib/skladchik-reports-monitor/cookies.jar",
|
||||
task.json| "ssh:debian@[PRIVATE_IP]:systemd:skladchik-reports-monitor.service",
|
||||
task.json| "ssh:debian@[PRIVATE_IP]:systemd:skladchik-reports-monitor.timer",
|
||||
task.json| "/usr/local/sbin/skladchik-reports-monitor-cookie-update",
|
||||
task.json| "/usr/local/sbin/skladchik-reports-monitor-backup",
|
||||
task.json| "/usr/local/sbin/skladchik-reports-monitor-monthly-selftest-watch",
|
||||
task.json| "/usr/local/sbin/skladchik-reports-monitor-status",
|
||||
task.json| "/usr/local/sbin/skladchik-reports-monitor-supervisor",
|
||||
task.json| "git:/srv/homelab-desired-state",
|
||||
task.json| "ssh:debian@[PRIVATE_IP]:/usr/local/sbin/skladchik-reports-monitor-cookie-update-edge"
|
||||
task.json| ],
|
||||
task.json| "expected_outputs": [
|
||||
task.json| {
|
||||
task.json| "path": "$HOMELAB_RUN_DIR/preflight.json",
|
||||
task.json| "producer": "phases/preflight.sh",
|
||||
task.json| "verification": "strict host-key access, the pinned live predecessor edge script, the versioned target edge script, AUTH_REQUIRED and bounded monitor quiescence are confirmed; evidence records both SHAs"
|
||||
task.json| },
|
||||
task.json| {
|
||||
task.json| "path": "$HOMELAB_RUN_DIR/prepare.json",
|
||||
task.json| "producer": "phases/prepare.sh",
|
||||
task.json| "verification": "root-only local and edge rollback snapshots have self-contained SHA256 manifests"
|
||||
task.json| },
|
||||
task.json| {
|
||||
task.json| "path": "$HOMELAB_RUN_DIR/dry-run.json",
|
||||
task.json| "producer": "phases/dry-run.sh",
|
||||
task.json| "verification": "rollback manifests validate, the pre-apply live edge script remains the pinned predecessor, the versioned deployment asset matches the pinned target SHA256, and evidence records both SHAs"
|
||||
task.json| },
|
||||
task.json| {
|
||||
task.json| "path": "$HOMELAB_RUN_DIR/apply.json",
|
||||
task.json| "producer": "phases/apply.sh",
|
||||
task.json| "verification": "apply replaces the pinned predecessor edge script with the pinned target script, quiesces the timer, performs one controlled reauthentication and restores or enables the timer"
|
||||
task.json| },
|
||||
task.json| {
|
||||
task.json| "path": "$HOMELAB_RUN_DIR/verify.json",
|
||||
task.json| "producer": "phases/verify.sh",
|
||||
task.json| "verification": "edge monitor, counts, timer, supervisor, backup and restore are OK"
|
||||
task.json| },
|
||||
task.json| {
|
||||
task.json| "path": "$HOMELAB_RUN_DIR/rollback.json",
|
||||
task.json| "producer": "phases/rollback.sh",
|
||||
task.json| "verification": "previous auth artifacts and timer state are restored"
|
||||
task.json| },
|
||||
task.json| {
|
||||
task.json| "path": "$HOMELAB_RUN_DIR/seal.json",
|
||||
task.json| "producer": "phases/seal.sh",
|
||||
task.json| "verification": "standard seal exists and rollback snapshot is removed"
|
||||
task.json| }
|
||||
task.json| ],
|
||||
task.json| "runtime_matrix": [
|
||||
task.json| {
|
||||
task.json| "host": "pve01.gram1.ru",
|
||||
task.json| "runtime": "bash,jq,git,systemd,ssh",
|
||||
task.json| "compile": "bash -n; jq -e"
|
||||
task.json| },
|
||||
task.json| {
|
||||
task.json| "host": "edge",
|
||||
task.json| "runtime": "bash,python3,systemd,sha256sum",
|
||||
task.json| "compile": "bash -n"
|
||||
task.json| }
|
||||
task.json| ],
|
||||
task.json| "rollback": {
|
||||
task.json| "command": "HOMELAB_CONTROLLED_APPLY_CONFIRM=skladchik-reports-monitor-controlled-reauth-v1 /opt/homelab-admin/bin/homelab-admin run TASK_DIR rollback",
|
||||
task.json| "verification": "self-contained local and edge manifests validate; prior auth artifacts, scripts, desired-state HEAD, supervisor state and timer state are restorable"
|
||||
task.json| },
|
||||
task.json| "no_change_assertions": [
|
||||
task.json| "vm160_backup_job",
|
||||
task.json| "cluster_backup_jobs",
|
||||
task.json| "unrelated_guests",
|
||||
task.json| "network",
|
||||
task.json| "dns",
|
||||
task.json| "known_hosts",
|
||||
task.json| "mail_cloud",
|
||||
task.json| "gitea",
|
||||
task.json| "backup_and_restore_artifacts"
|
||||
task.json| ],
|
||||
task.json| "source_of_truth": {
|
||||
task.json| "change_id": "CR-2026-0012",
|
||||
task.json| "incident": "Skladchik monitor AUTH_REQUIRED with missing complaint counts",
|
||||
task.json| "edge_script_sha256": "10fe854274dca1a053cfd8c0155340bd689bf3f6555c70dbc14b890cbafb93fa",
|
||||
task.json| "edge_script_path": "/usr/local/sbin/skladchik-reports-monitor-cookie-update-edge",
|
||||
task.json| "unsafe_legacy_wrapper_path": "/usr/local/sbin/skladchik-reports-monitor-cookie-update",
|
||||
task.json| "unsafe_legacy_wrapper_reason": "legacy host-key auto-acceptance",
|
||||
task.json| "strict_hostkey_required": true,
|
||||
task.json| "hidden_interactive_prompt_required": true,
|
||||
task.json| "cookie_content_must_not_enter_git_or_logs": true,
|
||||
task.json| "backup_status_before": "OK",
|
||||
task.json| "restore_status_before": "OK",
|
||||
task.json| "regression_lock": "REG-024-SKLADCHIK-REAUTH-STRICT-HOSTKEY",
|
||||
task.json| "active_pve01_script_count": 5,
|
||||
task.json| "active_pve01_original_sha256": {
|
||||
task.json| "skladchik-reports-monitor-cookie-update": "a3006723d902fb7735c593003a6c6755fa3d69683126ec5f53165131f754fd7a",
|
||||
task.json| "skladchik-reports-monitor-backup": "ad0652d1e154fca94a20bdf831b40b831fd0d862eb104ec3377f486e96270d0a",
|
||||
task.json| "skladchik-reports-monitor-monthly-selftest-watch": "dd2c7e26aaa6835fa4ac4365eade338eb7b19f4bfe7528179c59cf8039454180",
|
||||
task.json| "skladchik-reports-monitor-status": "f94ec75155f415b35517dade0bfba00926eba6088e0c2b5c96ae79daa5dc368f",
|
||||
task.json| "skladchik-reports-monitor-supervisor": "69494a4c32fa632844cdb54aabbe302a14d478924efcb719c6cd3d594d102369"
|
||||
task.json| },
|
||||
task.json| "active_pve01_deployed_sha256": {
|
||||
task.json| "skladchik-reports-monitor-cookie-update": "ea2d89ce67893a6d89b29610e4d46e207932ecc157173454585f67890f58810c",
|
||||
task.json| "skladchik-reports-monitor-backup": "1912f8f0559637563972bb8ac984fb0036694092056ae53968c02cc3b67855f3",
|
||||
task.json| "skladchik-reports-monitor-monthly-selftest-watch": "48c6ca01d2430d14c6af797c079640815ba1352a98466d4014ecbb80a27eaada",
|
||||
task.json| "skladchik-reports-monitor-status": "f902a61109efbac5918360f2235c2523bd10704ac484a4e8591e8a1c490faf5a",
|
||||
task.json| "skladchik-reports-monitor-supervisor": "35c68ae0929ff3795be2bde40de334fc8e26413a3d8a620a137be90d1f2c164e"
|
||||
task.json| },
|
||||
task.json| "edge_mutation_script_versioned": true,
|
||||
task.json| "desired_state_script_count_after_apply": 5,
|
||||
task.json| "all_active_skladchik_ssh_requires_strict_known_hosts": true,
|
||||
task.json| "unsafe_active_hostkey_markers_after_verify": 0,
|
||||
task.json| "pve01_assets_checked_by_repository_tests": true,
|
||||
task.json| "regression_locks": [
|
||||
task.json| "REG-024-SKLADCHIK-REAUTH-STRICT-HOSTKEY",
|
||||
task.json| "REG-025-EXACT-SHA256-LENGTH-VALIDATION",
|
||||
task.json| "REG-026-ACTIVE-SKLADCHIK-STRICT-HOSTKEY-COVERAGE",
|
||||
task.json| "REG-027-REMOTE-MUTATION-SCRIPT-MUST-BE-VERSIONED",
|
||||
task.json| "REG-028-FORBIDDEN-TOKEN-IN-NEGATIVE-ASSERTION",
|
||||
task.json| "REG-029-LEGACY-WRAPPER-NOT-COPIED-VERBATIM",
|
||||
task.json| "REG-030-BINARY-ARTIFACT-TEXT-SCAN",
|
||||
task.json| "REG-031-SEMANTIC-TEST-LOOP-EXPANSION-MISMATCH",
|
||||
task.json| "REG-032-OVERBROAD-INLINE-PYTHON-TEST",
|
||||
task.json| "REG-033-INSTALLED-SOURCE-MUST-BE-DECLARED",
|
||||
task.json| "REG-034-ROLLBACK-STAGE-PARENT-MODE",
|
||||
RUN_EVIDENCE_END
|
||||
CURRENT_FACTS_BEGIN
|
||||
EDGE_SCRIPT_LOCAL_EXISTS=true
|
||||
EDGE_SCRIPT_LOCAL_META=mode=700|uid=0|gid=0|size=5247
|
||||
EDGE_SCRIPT_LOCAL_SHA256=10fe854274dca1a053cfd8c0155340bd689bf3f6555c70dbc14b890cbafb93fa
|
||||
LOCAL_HEALTH_EXISTS=false
|
||||
SECRETS_ENV_EXISTS=false
|
||||
MONITOR_PY_EXISTS=false
|
||||
UNIT=skladchik-reports-monitor.service|active=inactive|enabled=not-found
|
||||
UNIT=skladchik-reports-monitor.timer|active=inactive|enabled=not-found
|
||||
UNIT=skladchik-reports-monitor-supervisor.service|active=failed|enabled=static
|
||||
UNIT=skladchik-reports-monitor-supervisor.timer|active=active|enabled=enabled
|
||||
CURRENT_FACTS_END
|
||||
RCA_CAPTURE_COMPLETE=true
|
||||
CHANGES_MADE_BY_943=false
|
||||
ROLLBACK_STARTED=false
|
||||
ROLLBACK_RESTORED=null
|
||||
HOMELAB_RESULT_CONTRACT={"version":1,"command_id":"SUPPORT-260923-SKLADCHIK-COOKIE-REFRESH-REISSUE-942R1","status":"FAIL","changes_made":true,"rollback_started":false,"rollback_restored":null}
|
||||
SKLADCHIK942_END=true
|
||||
HOMELAB_RESULT_CONTRACT={"version":1,"command_id":"SUPPORT-260923-SKLADCHIK-PREFLIGHT-LINE13-RCA-943R1","status":"OK","changes_made":false,"rollback_started":false,"rollback_restored":null}
|
||||
SKLADCHIK943_END=true
|
||||
|
||||
OUTPUT_END
|
||||
CHAT_OUTPUT_END
|
||||
|
||||
Reference in New Issue
Block a user