diff --git a/runtime/history/SUPPORT-260922-SPB-ACME-CERTBOT-FAIL-RCA-9124R1.json b/runtime/history/SUPPORT-260922-SPB-ACME-CERTBOT-FAIL-RCA-9124R1.json new file mode 100644 index 00000000..c76638ad --- /dev/null +++ b/runtime/history/SUPPORT-260922-SPB-ACME-CERTBOT-FAIL-RCA-9124R1.json @@ -0,0 +1,38 @@ +{ + "schema_version": 1, + "channel": "homelab-runtime", + "command_id": "SUPPORT-260922-SPB-ACME-CERTBOT-FAIL-RCA-9124R1", + "status": "OK", + "rc": 0, + "host": "pve01", + "mode": "read-only", + "component": "spb-acme-certbot-fail-rca", + "started_at_utc": "2026-09-22T14:09:59Z", + "finished_at_utc": "2026-09-22T14:10:04Z", + "reference_register_checked": true, + "reference_sha256": "5a3d8e5154c41cb582a4e0aca68090be1f0138918bf82131a948df326f9d8d66", + "error_register_checked": true, + "error_register_sha256": "3b09a553ec0f527ed3afeed4753f52a74ee3036045bbfb3c685e9f8af4ba7ba0", + "command_sha256": "65e45035e738859d7301cc86383acdbc514310a22e8a10f9a86928f593b10521", + "duplicate_failed_command_blocked": false, + "block_reason": null, + "execution_started": true, + "change_declared": false, + "result_contract_valid": true, + "result_contract_status": null, + "result_contract_error": null, + "command_rc": 0, + "changes_made": false, + "rollback_started": false, + "rollback_restored": null, + "mutation_outcome": "NO_MUTATION", + "sanitized": true, + "secrets_included": false, + "private_addresses_included": false, + "raw_evidence_retained_locally": true, + "raw_evidence_sha256": "0d656c5cf4c2e5febffda0117d173fb22f193ad50645d09c55e3eb5d4bbcb85c", + "sanitized_output_sha256": "0d656c5cf4c2e5febffda0117d173fb22f193ad50645d09c55e3eb5d4bbcb85c", + "output_truncated_in_json": false, + "full_sanitized_output_url": "https://git.gram1.ru/.well-known/homelab-runtime/latest.txt", + "output": "SPB9124_RCA={\"guest\":{\"ok\":true,\"data\":{\"hook\":{\"exists\":true,\"sha256\":\"dcaffc5a248359fa3ed93b7bf8af33ad69985776c4a8e3e7f2016c6168da5aee\",\"sha_matches_9120\":true,\"functions\":[\"out\",\"token\",\"api\",\"zone_id\",\"state_path\",\"auth\",\"cleanup\"],\"safe_to_import_without_main\":false,\"unsafe_top_level_types\":[\"Expr\"],\"selected_string_constants\":[\".json\",\"/dns_records\",\"/dns_records/\",\"/opt/npmplus/secure/gram1_cf_token\",\"/zones/\",\"/zones?\",\"GRAM1_CF_ZONE_LOOKUP ok=no http=\",\"GRAM1_CF_ZONE_LOOKUP ok=yes id_prefix=\",\"_acme-challenge.\",\"gram1.ru\",\"https://api.cloudflare.com/client/v4\",\"www.gram1.ru\",\"zone_id\"],\"function_sources\":{\"token\":[\"def token(): return TOKEN_PATH.read_text(encoding=\\\"utf-8\\\").strip()\"],\"api\":[\"def api(method, path, payload=None):\",\" data = json.dumps(payload).encode(\\\"utf-8\\\") if payload is not None else None\",\" req = urllib.request.Request(\\\"https://api.cloudflare.com/client/v4\\\" + path, data=data, headers={\\\"Authorization\\\": \\\"Bearer[REDACTED_EXPR]\\\" + token(), \\\"Content-Type\\\": \\\"application/json\\\"}, method=method)\",\" try:\",\" with urllib.request.urlopen(req, timeout=25) as r: return r.status, json.loads(r.read().decode(\\\"utf-8\\\", \\\"replace\\\"))\",\" except urllib.error.HTTPError as e:\",\" try: body = json.loads(e.read().decode(\\\"utf-8\\\", \\\"replace\\\"))\",\" except Exception: body = {\\\"success\\\": False, \\\"errors\\\": [{\\\"message\\\": \\\"http_error_non_json\\\"}]}\",\" return e.code, body\"],\"zone_id\":[\"def zone_id():\",\" status, data = api(\\\"GET\\\", \\\"/zones?\\\" + urllib.parse.urlencode({\\\"name\\\": ZONE}))\",\" if not data.get(\\\"success\\\") or not data.get(\\\"result\\\"):\",\" out(f\\\"GRAM1_CF_ZONE_LOOKUP ok=no http={status}\\\"); return None\",\" zid = data[\\\"result\\\"][0][\\\"id\\\"]; out(f\\\"GRAM1_CF_ZONE_LOOKUP ok=yes id_prefix={zid[:8]}\\\"); return zid\"],\"auth\":[\"def auth():\",\" domain = os.environ.get(\\\"CERTBOT_DOMAIN\\\", \\\"\\\"); validation = os.environ.get(\\\"CERTBOT_VALIDATION\\\", \\\"\\\")\",\" if domain not in (\\\"gram1.ru\\\", \\\"www.gram1.ru\\\") or not validation:\",\" out(f\\\"GRAM1_CF_AUTH domain={domain} status=review reason=bad_env\\\"); return 2\",\" zid = zone_id()\",\" if not zid: return 2\",\" name = \\\"_acme-challenge.\\\" + domain\",\" status, data = api(\\\"POST\\\", f\\\"/zones/{zid}/dns_records\\\", {\\\"type\\\": \\\"TXT\\\", \\\"name\\\": name, \\\"content\\\": validation, \\\"ttl\\\": 120})\",\" if not data.get(\\\"success\\\"):\",\" errs = data.get(\\\"errors\\\") or []; msg = str(errs[0].get(\\\"message\\\", \\\"\\\"))[:80].replace(\\\"\\\\n\\\", \\\" \\\") if errs else \\\"\\\"\",\" out(f\\\"GRAM1_CF_AUTH domain={domain} status=review http={status} error={msg}\\\"); return 2\",\" rec = data[\\\"result\\\"][\\\"id\\\"]; state_path(domain, validation).write_text(json.dumps({\\\"zone_id\\\": zid, \\\"record_id\\\": rec}), encoding=\\\"utf-8\\\")\",\" out(f\\\"GRAM1_CF_AUTH domain={domain} status=ok record_id_prefix={rec[:8]}\\\"); time.sleep(45); return 0\"],\"cleanup\":[\"def cleanup():\",\" domain = os.environ.get(\\\"CERTBOT_DOMAIN\\\", \\\"\\\"); validation = os.environ.get(\\\"CERTBOT_VALIDATION\\\", \\\"\\\")\",\" sp = state_path(domain, validation)\",\" if sp.exists():\",\" try:\",\" data = json.loads(sp.read_text(encoding=\\\"utf-8\\\"))\",\" status, resp = api(\\\"DELETE\\\", f\\\"/zones/{data['zone_id']}/dns_records/{data['record_id']}\\\")\",\" out(f\\\"GRAM1_CF_CLEANUP domain={domain} status={'ok' if resp.get('success') else 'review'} http={status}\\\"); sp.unlink(missing_ok=True)\",\" except Exception as e: out(f\\\"GRAM1_CF_CLEANUP domain={domain} status=review error={type(e).__name__}\\\")\",\" else: out(f\\\"GRAM1_CF_CLEANUP domain={domain} status=skip reason=no_state\\\")\",\" return 0\"]}},\"cloudflare_readonly\":{},\"certbot\":{\"data_mount_found\":true,\"data_mount_hash\":\"ec0ac67820f88633\",\"config_exists\":true,\"config_mode\":\"0o700\",\"account_json_count\":6,\"renewal_count\":13,\"renewal_basenames\":[\"hbarhub.ru.conf\",\"newfi-staging-gram1-ru.conf\",\"newfilya.ru.conf\",\"npm-17.conf\",\"npm-21.conf\",\"npm-26.conf\",\"npm-27.conf\",\"npm-29.conf\",\"npm-30.conf\",\"npm-31.conf\",\"npm-32.conf\",\"npm-35.conf\",\"npm-38.conf\"],\"certificates_rc\":0,\"certificates_diag\":[\"Renewal configuration file [PATH] produced an unexpected error: expected [PATH] to be a symlink. Skipping.\",\"Renewal configuration file [PATH] produced an unexpected error: expected [PATH] to be a symlink. Skipping.\",\"Renewal configuration file [PATH] produced an unexpected error: expected [PATH] to be a symlink. Skipping.\",\"Renewal configuration file [PATH] produced an unexpected error: expected [PATH] to be a symlink. Skipping.\",\"Renewal configuration file [PATH] produced an unexpected error: expected [PATH] to be a symlink. Skipping.\",\"Renewal configuration file [PATH] produced an unexpected error: expected [PATH] to be a symlink. Skipping.\",\"Renewal configuration file [PATH] produced an unexpected error: expected [PATH] to be a symlink. Skipping.\",\"Renewal configuration file [PATH] produced an unexpected error: expected [PATH] to be a symlink. Skipping.\",\"Renewal configuration file [PATH] produced an unexpected error: expected [PATH] to be a symlink. Skipping.\",\"Renewal configuration file [PATH] produced an unexpected error: expected [PATH] to be a symlink. Skipping.\",\"Renewal configuration file [PATH] produced an unexpected error: expected [PATH] to be a symlink. Skipping.\",\"Renewal configuration file [PATH] produced an unexpected error: expected [PATH] to be a symlink. Skipping.\",\"Renewal configuration file [PATH] produced an unexpected error: expected [PATH] to be a symlink. Skipping.\",\"The following renewal configurations were invalid:\",\" [PATH]\",\" [PATH]\",\" [PATH]\",\" [PATH]\",\" [PATH]\",\" [PATH]\",\" [PATH]\",\" [PATH]\",\" [PATH]\",\" [PATH]\",\" [PATH]\",\" [PATH]\",\" [PATH]\"],\"certificates_stderr_sha\":\"d744a29aaabe2413\",\"show_account_rc\":1,\"show_account_diag\":[\"An unexpected error occurred:\",\"ValueError: Requesting acme-v02.api.letsencrypt.org/directory: No route to host\"],\"gitread_live_exists\":false,\"gitread_renewal_exists\":false,\"new_vhost_files_exist\":[]},\"locks\":{\"present\":[]},\"processes\":{\"certbot_related\":[]}}}}\nTASK_COMPLETE=true\nTASK_RESULT=PASS_SPB_ACME_CERTBOT_FAIL_RCA\nDECISION=READ_EXACT_9124_THEN_CORRECT_ACME_EXECUTION_PATH\n" +} diff --git a/runtime/history/SUPPORT-260922-SPB-ACME-CERTBOT-FAIL-RCA-9124R1.txt b/runtime/history/SUPPORT-260922-SPB-ACME-CERTBOT-FAIL-RCA-9124R1.txt new file mode 100644 index 00000000..7e4e1aa1 --- /dev/null +++ b/runtime/history/SUPPORT-260922-SPB-ACME-CERTBOT-FAIL-RCA-9124R1.txt @@ -0,0 +1,36 @@ +CHAT_OUTPUT_BEGIN +COMMAND_ID=SUPPORT-260922-SPB-ACME-CERTBOT-FAIL-RCA-9124R1 +STATUS=OK +RC=0 +HOST=pve01 +MODE=read-only +COMPONENT=spb-acme-certbot-fail-rca +REFERENCE_REGISTER_CHECK=OK +REFERENCE_SHA256=5a3d8e5154c41cb582a4e0aca68090be1f0138918bf82131a948df326f9d8d66 +ERROR_REGISTER_CHECK=OK +ERROR_REGISTER_SHA256=3b09a553ec0f527ed3afeed4753f52a74ee3036045bbfb3c685e9f8af4ba7ba0 +COMMAND_SHA256=65e45035e738859d7301cc86383acdbc514310a22e8a10f9a86928f593b10521 +DUPLICATE_FAILED_COMMAND_BLOCKED=false +EXECUTION_STARTED=true +CHANGE_DECLARED=false +RESULT_CONTRACT_VALID=true +RESULT_CONTRACT_STATUS=NOT_APPLICABLE +RESULT_CONTRACT_ERROR=NONE +COMMAND_RC=0 +CHANGES_MADE=false +ROLLBACK_STARTED=false +ROLLBACK_RESTORED=null +MUTATION_OUTCOME=NO_MUTATION +SANITIZED=yes +SECRETS_INCLUDED=no +PRIVATE_ADDRESSES_INCLUDED=no +RAW_EVIDENCE_SHA256=0d656c5cf4c2e5febffda0117d173fb22f193ad50645d09c55e3eb5d4bbcb85c +SANITIZED_OUTPUT_SHA256=0d656c5cf4c2e5febffda0117d173fb22f193ad50645d09c55e3eb5d4bbcb85c +OUTPUT_BEGIN +SPB9124_RCA={"guest":{"ok":true,"data":{"hook":{"exists":true,"sha256":"dcaffc5a248359fa3ed93b7bf8af33ad69985776c4a8e3e7f2016c6168da5aee","sha_matches_9120":true,"functions":["out","token","api","zone_id","state_path","auth","cleanup"],"safe_to_import_without_main":false,"unsafe_top_level_types":["Expr"],"selected_string_constants":[".json","/dns_records","/dns_records/","/opt/npmplus/secure/gram1_cf_token","/zones/","/zones?","GRAM1_CF_ZONE_LOOKUP ok=no http=","GRAM1_CF_ZONE_LOOKUP ok=yes id_prefix=","_acme-challenge.","gram1.ru","https://api.cloudflare.com/client/v4","www.gram1.ru","zone_id"],"function_sources":{"token":["def token(): return TOKEN_PATH.read_text(encoding=\"utf-8\").strip()"],"api":["def api(method, path, payload=None):"," data = json.dumps(payload).encode(\"utf-8\") if payload is not None else None"," req = urllib.request.Request(\"https://api.cloudflare.com/client/v4\" + path, data=data, headers={\"Authorization\": \"Bearer[REDACTED_EXPR]\" + token(), \"Content-Type\": \"application/json\"}, method=method)"," try:"," with urllib.request.urlopen(req, timeout=25) as r: return r.status, json.loads(r.read().decode(\"utf-8\", \"replace\"))"," except urllib.error.HTTPError as e:"," try: body = json.loads(e.read().decode(\"utf-8\", \"replace\"))"," except Exception: body = {\"success\": False, \"errors\": [{\"message\": \"http_error_non_json\"}]}"," return e.code, body"],"zone_id":["def zone_id():"," status, data = api(\"GET\", \"/zones?\" + urllib.parse.urlencode({\"name\": ZONE}))"," if not data.get(\"success\") or not data.get(\"result\"):"," out(f\"GRAM1_CF_ZONE_LOOKUP ok=no http={status}\"); return None"," zid = data[\"result\"][0][\"id\"]; out(f\"GRAM1_CF_ZONE_LOOKUP ok=yes id_prefix={zid[:8]}\"); return zid"],"auth":["def auth():"," domain = os.environ.get(\"CERTBOT_DOMAIN\", \"\"); validation = os.environ.get(\"CERTBOT_VALIDATION\", \"\")"," if domain not in (\"gram1.ru\", \"www.gram1.ru\") or not validation:"," out(f\"GRAM1_CF_AUTH domain={domain} status=review reason=bad_env\"); return 2"," zid = zone_id()"," if not zid: return 2"," name = \"_acme-challenge.\" + domain"," status, data = api(\"POST\", f\"/zones/{zid}/dns_records\", {\"type\": \"TXT\", \"name\": name, \"content\": validation, \"ttl\": 120})"," if not data.get(\"success\"):"," errs = data.get(\"errors\") or []; msg = str(errs[0].get(\"message\", \"\"))[:80].replace(\"\\n\", \" \") if errs else \"\""," out(f\"GRAM1_CF_AUTH domain={domain} status=review http={status} error={msg}\"); return 2"," rec = data[\"result\"][\"id\"]; state_path(domain, validation).write_text(json.dumps({\"zone_id\": zid, \"record_id\": rec}), encoding=\"utf-8\")"," out(f\"GRAM1_CF_AUTH domain={domain} status=ok record_id_prefix={rec[:8]}\"); time.sleep(45); return 0"],"cleanup":["def cleanup():"," domain = os.environ.get(\"CERTBOT_DOMAIN\", \"\"); validation = os.environ.get(\"CERTBOT_VALIDATION\", \"\")"," sp = state_path(domain, validation)"," if sp.exists():"," try:"," data = json.loads(sp.read_text(encoding=\"utf-8\"))"," status, resp = api(\"DELETE\", f\"/zones/{data['zone_id']}/dns_records/{data['record_id']}\")"," out(f\"GRAM1_CF_CLEANUP domain={domain} status={'ok' if resp.get('success') else 'review'} http={status}\"); sp.unlink(missing_ok=True)"," except Exception as e: out(f\"GRAM1_CF_CLEANUP domain={domain} status=review error={type(e).__name__}\")"," else: out(f\"GRAM1_CF_CLEANUP domain={domain} status=skip reason=no_state\")"," return 0"]}},"cloudflare_readonly":{},"certbot":{"data_mount_found":true,"data_mount_hash":"ec0ac67820f88633","config_exists":true,"config_mode":"0o700","account_json_count":6,"renewal_count":13,"renewal_basenames":["hbarhub.ru.conf","newfi-staging-gram1-ru.conf","newfilya.ru.conf","npm-17.conf","npm-21.conf","npm-26.conf","npm-27.conf","npm-29.conf","npm-30.conf","npm-31.conf","npm-32.conf","npm-35.conf","npm-38.conf"],"certificates_rc":0,"certificates_diag":["Renewal configuration file [PATH] produced an unexpected error: expected [PATH] to be a symlink. Skipping.","Renewal configuration file [PATH] produced an unexpected error: expected [PATH] to be a symlink. Skipping.","Renewal configuration file [PATH] produced an unexpected error: expected [PATH] to be a symlink. Skipping.","Renewal configuration file [PATH] produced an unexpected error: expected [PATH] to be a symlink. Skipping.","Renewal configuration file [PATH] produced an unexpected error: expected [PATH] to be a symlink. Skipping.","Renewal configuration file [PATH] produced an unexpected error: expected [PATH] to be a symlink. Skipping.","Renewal configuration file [PATH] produced an unexpected error: expected [PATH] to be a symlink. Skipping.","Renewal configuration file [PATH] produced an unexpected error: expected [PATH] to be a symlink. Skipping.","Renewal configuration file [PATH] produced an unexpected error: expected [PATH] to be a symlink. Skipping.","Renewal configuration file [PATH] produced an unexpected error: expected [PATH] to be a symlink. Skipping.","Renewal configuration file [PATH] produced an unexpected error: expected [PATH] to be a symlink. Skipping.","Renewal configuration file [PATH] produced an unexpected error: expected [PATH] to be a symlink. Skipping.","Renewal configuration file [PATH] produced an unexpected error: expected [PATH] to be a symlink. Skipping.","The following renewal configurations were invalid:"," [PATH]"," [PATH]"," [PATH]"," [PATH]"," [PATH]"," [PATH]"," [PATH]"," [PATH]"," [PATH]"," [PATH]"," [PATH]"," [PATH]"," [PATH]"],"certificates_stderr_sha":"d744a29aaabe2413","show_account_rc":1,"show_account_diag":["An unexpected error occurred:","ValueError: Requesting acme-v02.api.letsencrypt.org/directory: No route to host"],"gitread_live_exists":false,"gitread_renewal_exists":false,"new_vhost_files_exist":[]},"locks":{"present":[]},"processes":{"certbot_related":[]}}}} +TASK_COMPLETE=true +TASK_RESULT=PASS_SPB_ACME_CERTBOT_FAIL_RCA +DECISION=READ_EXACT_9124_THEN_CORRECT_ACME_EXECUTION_PATH + +OUTPUT_END +CHAT_OUTPUT_END diff --git a/runtime/latest.json b/runtime/latest.json index 6ac14e23..c76638ad 100644 --- a/runtime/latest.json +++ b/runtime/latest.json @@ -1,19 +1,19 @@ { "schema_version": 1, "channel": "homelab-runtime", - "command_id": "NEWFI-260922-NEWFDOM-DR221-VM210-PRODUCER-MEMBERSHIP-RCA416R1", + "command_id": "SUPPORT-260922-SPB-ACME-CERTBOT-FAIL-RCA-9124R1", "status": "OK", "rc": 0, "host": "pve01", "mode": "read-only", - "component": "newfi-dr221-vm210-producer-membership-rca", - "started_at_utc": "2026-09-22T14:05:20Z", - "finished_at_utc": "2026-09-22T14:05:20Z", + "component": "spb-acme-certbot-fail-rca", + "started_at_utc": "2026-09-22T14:09:59Z", + "finished_at_utc": "2026-09-22T14:10:04Z", "reference_register_checked": true, "reference_sha256": "5a3d8e5154c41cb582a4e0aca68090be1f0138918bf82131a948df326f9d8d66", "error_register_checked": true, "error_register_sha256": "3b09a553ec0f527ed3afeed4753f52a74ee3036045bbfb3c685e9f8af4ba7ba0", - "command_sha256": "f147173d66cd34c3f641e8e304fe06d95889d3fb0886845c883d37b7494f102f", + "command_sha256": "65e45035e738859d7301cc86383acdbc514310a22e8a10f9a86928f593b10521", "duplicate_failed_command_blocked": false, "block_reason": null, "execution_started": true, @@ -30,9 +30,9 @@ "secrets_included": false, "private_addresses_included": false, "raw_evidence_retained_locally": true, - "raw_evidence_sha256": "df560dcc7c495f7172790e5839527d5a14299e541ad404e0fee0a82517a452eb", - "sanitized_output_sha256": "4c78ca1b8648c6cbd612ae432ff927d18627ba6573375c6afc3af25de784b531", + "raw_evidence_sha256": "0d656c5cf4c2e5febffda0117d173fb22f193ad50645d09c55e3eb5d4bbcb85c", + "sanitized_output_sha256": "0d656c5cf4c2e5febffda0117d173fb22f193ad50645d09c55e3eb5d4bbcb85c", "output_truncated_in_json": false, "full_sanitized_output_url": "https://git.gram1.ru/.well-known/homelab-runtime/latest.txt", - "output": "NEWFI416_BEGIN=true\nCOMMAND_ID=NEWFI-260922-NEWFDOM-DR221-VM210-PRODUCER-MEMBERSHIP-RCA416R1\nMODE=read-only\nCOMPONENT=newfi-dr221-vm210-producer-membership-rca\nSCOPE=PROVE_OR_REFUTE_VM210_MEMBERSHIP_IN_CANONICAL_VM_CLOUD_QUORUM_PRODUCER\nMUTATION_BOUNDARY=NONE;NO_BACKUP;NO_DISPATCH;NO_SYSTEMD_ACTION;NO_CLOUD_WRITE;NO_VM_WRITE;NO_GIT_WRITE\nREFERENCE_SHA256=5a3d8e5154c41cb582a4e0aca68090be1f0138918bf82131a948df326f9d8d66\nERROR_REGISTER_SHA256=3b09a553ec0f527ed3afeed4753f52a74ee3036045bbfb3c685e9f8af4ba7ba0\nRUNNER_SHA256=b248a4c32c9cc64e5747e7dce6c7fc0a23f5124a77c71ce72e27a81aceae9d2d\nQUEUE_SHA256=fbc27ad67fc3d3f51934e1d73180fe0df8c9d15f129f34ebba39c56aa1d03612\nFRESHNESS_SHA256=bd6257397c6ea910df2ea0dd57252b33bccd0bbd965175840486dfe5e079bf96\nNODE_WORKER_SHA256=91bdd116be865560a38105dffbd1a66d687232035e2359e4a5be4d225e9c61d1\nVM210_ALL_TARGET_ROWS=2\nVM210_PVE_TARGET_COUNT=1\nVM210_XF_TARGET_COUNT=1\nVM210_PVE_ENABLED=true\nVM210_XF_ENABLED=true\nPRODUCER_MEMBERSHIP_SOURCE_BEGIN=true\nPRODUCER_FILE=/usr/local/sbin/homelab-vm-cloud-quorum-queue|SHA256=fbc27ad67fc3d3f51934e1d73180fe0df8c9d15f129f34ebba39c56aa1d03612\n9-FRESHNESS_SECONDS=72000\n10-DUE_SECONDS=43200\n11-SSH_BASE=(-o BatchMode=yes -o ConnectTimeout=12 -o StrictHostKeyChecking=yes -o UserKnownHostsFile=[SENSITIVE_PATH]\n12:ALL_VMIDS=(160 170 130 171 180 190 150 9130 110 111 112 113 200)\n13-MIGRATE_VMIDS=(160 170 130 171 180 190 150 9130 111 112 113)\n14-\n15-die(){ printf 'VM_QUORUM_QUEUE_ERROR=%s\\n' \"$1\" >&2; exit \"${2:-65}\"; }\n16-(( DUE_SECONDS > 0 && DUE_SECONDS < FRESHNESS_SECONDS )) || die DUE_THRESHOLD_INVALID 65\n17:expected_owner(){\n18- case \"$1\" in\n19- 110|112|150|170|171) printf 'pve01\\n' ;;\n20- 111|113|160|180|9130) printf 'pve02\\n' ;;\n21- 130|190|200) printf 'pve03\\n' ;;\n22: *) die \"VMID_NOT_ALLOWED_$1\" 64 ;;\n23- esac\n24-}\n25-observed_owner(){\n--\n45-}\n46-_run_worker_unlocked() {\n47- local vmid=\"$1\" action=\"$2\" generation=\"${3:-}\" owner observed host\n48: owner=\"$(expected_owner \"$vmid\")\"\n49- observed=\"$(observed_owner \"$vmid\")\"\n50- [[ \"$observed\" == \"$owner\" ]] || die \"CLUSTER_OWNER_MISMATCH_${vmid}_${observed}_EXPECTED_${owner}\" 65\n51- if [[ \"$owner\" == \"pve01\" ]]; then\n--\n61-}\n62-run_probe(){\n63- local vmid=\"$1\" owner observed host\n64: owner=\"$(expected_owner \"$vmid\")\"\n65- observed=\"$(observed_owner \"$vmid\")\"\n66- [[ \"$observed\" == \"$owner\" ]] || die \"CLUSTER_OWNER_MISMATCH_${vmid}_${observed}_EXPECTED_${owner}\" 65\n67- if [[ \"$owner\" == \"pve01\" ]]; then\n--\n73-}\n74-plan_all(){\n75- local vmid\n76: for vmid in \"${ALL_VMIDS[@]}\"; do\n77: printf 'QUEUE_PLAN VMID=%s EXPECTED_OWNER=%s OBSERVED_OWNER=%s\\n' \"$vmid\" \"$(expected_owner \"$vmid\")\" \"$(observed_owner \"$vmid\")\"\n78- done\n79-}\n80-freshness_decision(){\n--\n116-freshness_plan(){\n117- local vmid now_epoch out decision\n118- [[ -x \"$PROBE\" ]] || die \"FRESHNESS_PROBE_NOT_INSTALLED\" 69\n119: for vmid in \"${ALL_VMIDS[@]}\"; do\n120- now_epoch=\"$(date -u +%s)\"\n121- out=\"$(freshness_decision \"$vmid\" \"$now_epoch\")\"\n122- printf '%s\\n' \"$out\"\n--\n132- exec 8>\"$HEAVY_LOCK\"\n133- flock -n 8 || die \"GLOBAL_HEAVY_LOCK_BUSY\" 75\n134- for vmid in \"$@\"; do\n135: printf 'QUEUE_BEGIN VMID=%s OWNER=%s MODE=FORCE\\n' \"$vmid\" \"$(expected_owner \"$vmid\")\"\n136- out=\"$(exec 8>&-; exec 9>&-; run_worker \"$vmid\" backup)\"\n137- printf '%s\\n' \"$out\"\n138- generation=\"$(awk -F= '$1==\"GENERATION\"{print $2}' <<<\"$out\" | tail -1)\"\n--\n146-dispatch_one(){\n147- local vmid=\"$1\" freshness decision now_epoch out generation\n148- [[ \"${HOMELAB_BACKUP_DISPATCHER:-}\" == 1 ]] || die \"DISPATCH_ONE_DIRECT_FORBIDDEN\" 64\n149: expected_owner \"$vmid\" >/dev/null\n150- exec 9>\"$LOCK\"\n151- if ! flock -n 9; then\n152- printf 'QUEUE_DEFERRED KIND=VM REASON=GLOBAL_QUEUE_LOCK_BUSY VMID=%s\\n' \"$vmid\"\n--\n170- BACKUP_RESUME|BACKUP_STALE|BACKUP_DUE|BACKUP_UNVERIFIED) ;;\n171- *) die \"FRESHNESS_RECHECK_INVALID_${vmid}_${decision:-EMPTY}\" 65 ;;\n172- esac\n173: printf 'QUEUE_BEGIN VMID=%s OWNER=%s MODE=%s DISPATCH=ONE_TARGET\\n' \"$vmid\" \"$(expected_owner \"$vmid\")\" \"$decision\"\n174- out=\"$(exec 8>&-; exec 9>&-; run_worker \"$vmid\" backup)\"\n175- printf '%s\\n' \"$out\"\n176- generation=\"$(awk -F= '$1==\"GENERATION\"{print $2}' <<<\"$out\" | tail -1)\"\nPRODUCER_FILE=/usr/local/libexec/homelab-vm-cloud-quorum-freshness-probe|SHA256=bd6257397c6ea910df2ea0dd57252b33bccd0bbd965175840486dfe5e079bf96\n5-RESTORE_ROOT=/var/lib/homelab-backup/vm-restore-validation\n6-HOST=\"$(hostname -s)\"\n7-die(){ printf 'VM_FRESHNESS_PROBE_ERROR=%s\\n' \"$1\" >&2; exit \"${2:-65}\"; }\n8:expected_owner(){\n9-case \"$1\" in\n10-110|112|150|170|171) printf 'pve01\\n' ;;\n11-111|113|160|180|9130) printf 'pve02\\n' ;;\n12-130|190|200) printf 'pve03\\n' ;;\n13:*) die \"VMID_NOT_ALLOWED_$1\" 64 ;;\n14-esac\n15-}\n16-assert_local_identity(){\n17-local vmid=\"$1\" owner\n18:owner=\"$(expected_owner \"$vmid\")\"\n19-[[ \"$HOST\" == \"$owner\" ]] || die \"OWNER_MISMATCH_${vmid}_${HOST}_EXPECTED_${owner}\" 65\n20-}\n21-state_scan(){\nPRODUCER_FILE=/usr/local/libexec/homelab-vm-cloud-quorum-node-worker|SHA256=91bdd116be865560a38105dffbd1a66d687232035e2359e4a5be4d225e9c61d1\n21-active_backup_restore(){ local rc; if pgrep -af '(^|/)(vzdump|qmrestore|pctrestore|vma|zstd|rclone)([[:space:]]|$)'; then return 0; else rc=$?; [[ \"$rc\" == 1 ]] && return 0; die \"PGREP_FAILED_$rc\" 69; fi; }\n22-snapshot_count_160(){ qm listsnapshot 160 | awk '/->/ && $0 !~ /current/{n++} END{print n+0}'; }\n23-vm_lock(){ qm config \"$1\" | awk -F': ' '$1==\"lock\"{print $2; found=1} END{if(!found) print \"none\"}'; }\n24:expected_owner(){\n25-case \"$1\" in\n26-110|112|150|170|171) printf 'pve01\\n' ;;\n27-111|113|160|180|9130) printf 'pve02\\n' ;;\n28-130|190|200) printf 'pve03\\n' ;;\n29:*) die \"VMID_NOT_ALLOWED_$1\" 64 ;;\n30-esac\n31-}\n32:expected_type(){\n33-case \"$1\" in\n34:110|111|112|113|200) printf 'lxc\\n' ;;\n35:130|150|160|170|171|180|190|9130) printf 'qemu\\n' ;;\n36:*) die \"VMID_NOT_ALLOWED_$1\" 64 ;;\n37-esac\n38-}\n39-expected_status(){\n--\n45-}\n46-assert_local_identity(){\n47-local vmid=\"$1\" owner type status expect_status\n48:owner=\"$(expected_owner \"$vmid\")\"\n49:type=\"$(expected_type \"$vmid\")\"\n50-[[ \"$HOST\" == \"$owner\" ]] || die \"OWNER_MISMATCH_${vmid}_${HOST}_EXPECTED_${owner}\" 65\n51-if [[ \"$type\" == \"qemu\" ]]; then\n52-[[ -s \"/etc/pve/qemu-server/${vmid}.conf\" ]] || die \"QEMU_CONFIG_MISSING_$vmid\" 66\n--\n153-plan(){\n154-local vmid=\"$1\" owner type status\n155-assert_local_identity \"$vmid\"\n156:owner=\"$(expected_owner \"$vmid\")\"\n157:type=\"$(expected_type \"$vmid\")\"\n158-status=\"$(expected_status \"$vmid\")\"\n159-printf 'PLAN_STATUS=OK\\nVMID=%s\\nOWNER=%s\\nTYPE=%s\\nEXPECTED_STATUS=%s\\n' \"$vmid\" \"$owner\" \"$type\" \"$status\"\n160-}\nPRODUCER_MEMBERSHIP_SOURCE_END=true\nVM210_FRESHNESS_DIRECT_RC=64\nVM210_FRESHNESS_STDOUT_SHA256=e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855\nVM210_FRESHNESS_STDERR_SHA256=b0d9e4d7ad61d749dfaf399ab3ad606613113a88a2c3e7085ff17e2463dc8b9e\nVM210_FRESHNESS_NOT_ALLOWED=true\nQUEUE_PLAN_RC=0\nQUEUE_PLAN_HAS_VM210=false\nQUEUE_PLAN_STDERR_SHA256=e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855\nSTATIC_FILE=/usr/local/sbin/homelab-vm-cloud-quorum-queue\nSTATIC_CONTAINS_210_LITERAL=false\nSTATIC_CONTAINS_VM9130_LITERAL=true\nSTATIC_LINE=12:ALL_VMIDS=(160 170 130 171 180 190 150 9130 110 111 112 113 200)\nSTATIC_LINE=22: *) die \"VMID_NOT_ALLOWED_$1\" 64 ;;\nSTATIC_LINE=76: for vmid in \"${ALL_VMIDS[@]}\"; do\nSTATIC_LINE=119: for vmid in \"${ALL_VMIDS[@]}\"; do\nSTATIC_FILE=/usr/local/libexec/homelab-vm-cloud-quorum-freshness-probe\nSTATIC_CONTAINS_210_LITERAL=false\nSTATIC_CONTAINS_VM9130_LITERAL=true\nSTATIC_LINE=13:*) die \"VMID_NOT_ALLOWED_$1\" 64 ;;\nSTATIC_FILE=/usr/local/libexec/homelab-vm-cloud-quorum-node-worker\nSTATIC_CONTAINS_210_LITERAL=false\nSTATIC_CONTAINS_VM9130_LITERAL=true\nSTATIC_LINE=29:*) die \"VMID_NOT_ALLOWED_$1\" 64 ;;\nSTATIC_LINE=34:110|111|112|113|200) printf 'lxc\\n' ;;\nSTATIC_LINE=35:130|150|160|170|171|180|190|9130) printf 'qemu\\n' ;;\nSTATIC_LINE=36:*) die \"VMID_NOT_ALLOWED_$1\" 64 ;;\nOWNER_STATE|service=activating|sub=start|heavy_lock=BUSY\nDECISION=PASS_DR221_416_VM210_PRODUCER_MEMBERSHIP_MISSING_PROVEN\nNEXT_GATE=WAIT_OWNER_THEN_REPAIR_VM210_CLOUD_QUORUM_PRODUCER_MEMBERSHIP_AND_FINAL_DR\nCHANGES_MADE_BY_416=false\nPRODUCT_MUTATION_BY_416=false\nVM_MUTATION_BY_416=false\nCLOUD_MUTATION_BY_416=false\nHOMELAB_RESULT_CONTRACT={\"version\":1,\"command_id\":\"NEWFI-260922-NEWFDOM-DR221-VM210-PRODUCER-MEMBERSHIP-RCA416R1\",\"status\":\"OK\",\"changes_made\":false,\"rollback_started\":false,\"rollback_restored\":null}\nNEWFI416_END=true\n" + "output": "SPB9124_RCA={\"guest\":{\"ok\":true,\"data\":{\"hook\":{\"exists\":true,\"sha256\":\"dcaffc5a248359fa3ed93b7bf8af33ad69985776c4a8e3e7f2016c6168da5aee\",\"sha_matches_9120\":true,\"functions\":[\"out\",\"token\",\"api\",\"zone_id\",\"state_path\",\"auth\",\"cleanup\"],\"safe_to_import_without_main\":false,\"unsafe_top_level_types\":[\"Expr\"],\"selected_string_constants\":[\".json\",\"/dns_records\",\"/dns_records/\",\"/opt/npmplus/secure/gram1_cf_token\",\"/zones/\",\"/zones?\",\"GRAM1_CF_ZONE_LOOKUP ok=no http=\",\"GRAM1_CF_ZONE_LOOKUP ok=yes id_prefix=\",\"_acme-challenge.\",\"gram1.ru\",\"https://api.cloudflare.com/client/v4\",\"www.gram1.ru\",\"zone_id\"],\"function_sources\":{\"token\":[\"def token(): return TOKEN_PATH.read_text(encoding=\\\"utf-8\\\").strip()\"],\"api\":[\"def api(method, path, payload=None):\",\" data = json.dumps(payload).encode(\\\"utf-8\\\") if payload is not None else None\",\" req = urllib.request.Request(\\\"https://api.cloudflare.com/client/v4\\\" + path, data=data, headers={\\\"Authorization\\\": \\\"Bearer[REDACTED_EXPR]\\\" + token(), \\\"Content-Type\\\": \\\"application/json\\\"}, method=method)\",\" try:\",\" with urllib.request.urlopen(req, timeout=25) as r: return r.status, json.loads(r.read().decode(\\\"utf-8\\\", \\\"replace\\\"))\",\" except urllib.error.HTTPError as e:\",\" try: body = json.loads(e.read().decode(\\\"utf-8\\\", \\\"replace\\\"))\",\" except Exception: body = {\\\"success\\\": False, \\\"errors\\\": [{\\\"message\\\": \\\"http_error_non_json\\\"}]}\",\" return e.code, body\"],\"zone_id\":[\"def zone_id():\",\" status, data = api(\\\"GET\\\", \\\"/zones?\\\" + urllib.parse.urlencode({\\\"name\\\": ZONE}))\",\" if not data.get(\\\"success\\\") or not data.get(\\\"result\\\"):\",\" out(f\\\"GRAM1_CF_ZONE_LOOKUP ok=no http={status}\\\"); return None\",\" zid = data[\\\"result\\\"][0][\\\"id\\\"]; out(f\\\"GRAM1_CF_ZONE_LOOKUP ok=yes id_prefix={zid[:8]}\\\"); return zid\"],\"auth\":[\"def auth():\",\" domain = os.environ.get(\\\"CERTBOT_DOMAIN\\\", \\\"\\\"); validation = os.environ.get(\\\"CERTBOT_VALIDATION\\\", \\\"\\\")\",\" if domain not in (\\\"gram1.ru\\\", \\\"www.gram1.ru\\\") or not validation:\",\" out(f\\\"GRAM1_CF_AUTH domain={domain} status=review reason=bad_env\\\"); return 2\",\" zid = zone_id()\",\" if not zid: return 2\",\" name = \\\"_acme-challenge.\\\" + domain\",\" status, data = api(\\\"POST\\\", f\\\"/zones/{zid}/dns_records\\\", {\\\"type\\\": \\\"TXT\\\", \\\"name\\\": name, \\\"content\\\": validation, \\\"ttl\\\": 120})\",\" if not data.get(\\\"success\\\"):\",\" errs = data.get(\\\"errors\\\") or []; msg = str(errs[0].get(\\\"message\\\", \\\"\\\"))[:80].replace(\\\"\\\\n\\\", \\\" \\\") if errs else \\\"\\\"\",\" out(f\\\"GRAM1_CF_AUTH domain={domain} status=review http={status} error={msg}\\\"); return 2\",\" rec = data[\\\"result\\\"][\\\"id\\\"]; state_path(domain, validation).write_text(json.dumps({\\\"zone_id\\\": zid, \\\"record_id\\\": rec}), encoding=\\\"utf-8\\\")\",\" out(f\\\"GRAM1_CF_AUTH domain={domain} status=ok record_id_prefix={rec[:8]}\\\"); time.sleep(45); return 0\"],\"cleanup\":[\"def cleanup():\",\" domain = os.environ.get(\\\"CERTBOT_DOMAIN\\\", \\\"\\\"); validation = os.environ.get(\\\"CERTBOT_VALIDATION\\\", \\\"\\\")\",\" sp = state_path(domain, validation)\",\" if sp.exists():\",\" try:\",\" data = json.loads(sp.read_text(encoding=\\\"utf-8\\\"))\",\" status, resp = api(\\\"DELETE\\\", f\\\"/zones/{data['zone_id']}/dns_records/{data['record_id']}\\\")\",\" out(f\\\"GRAM1_CF_CLEANUP domain={domain} status={'ok' if resp.get('success') else 'review'} http={status}\\\"); sp.unlink(missing_ok=True)\",\" except Exception as e: out(f\\\"GRAM1_CF_CLEANUP domain={domain} status=review error={type(e).__name__}\\\")\",\" else: out(f\\\"GRAM1_CF_CLEANUP domain={domain} status=skip reason=no_state\\\")\",\" return 0\"]}},\"cloudflare_readonly\":{},\"certbot\":{\"data_mount_found\":true,\"data_mount_hash\":\"ec0ac67820f88633\",\"config_exists\":true,\"config_mode\":\"0o700\",\"account_json_count\":6,\"renewal_count\":13,\"renewal_basenames\":[\"hbarhub.ru.conf\",\"newfi-staging-gram1-ru.conf\",\"newfilya.ru.conf\",\"npm-17.conf\",\"npm-21.conf\",\"npm-26.conf\",\"npm-27.conf\",\"npm-29.conf\",\"npm-30.conf\",\"npm-31.conf\",\"npm-32.conf\",\"npm-35.conf\",\"npm-38.conf\"],\"certificates_rc\":0,\"certificates_diag\":[\"Renewal configuration file [PATH] produced an unexpected error: expected [PATH] to be a symlink. Skipping.\",\"Renewal configuration file [PATH] produced an unexpected error: expected [PATH] to be a symlink. Skipping.\",\"Renewal configuration file [PATH] produced an unexpected error: expected [PATH] to be a symlink. Skipping.\",\"Renewal configuration file [PATH] produced an unexpected error: expected [PATH] to be a symlink. Skipping.\",\"Renewal configuration file [PATH] produced an unexpected error: expected [PATH] to be a symlink. Skipping.\",\"Renewal configuration file [PATH] produced an unexpected error: expected [PATH] to be a symlink. Skipping.\",\"Renewal configuration file [PATH] produced an unexpected error: expected [PATH] to be a symlink. Skipping.\",\"Renewal configuration file [PATH] produced an unexpected error: expected [PATH] to be a symlink. Skipping.\",\"Renewal configuration file [PATH] produced an unexpected error: expected [PATH] to be a symlink. Skipping.\",\"Renewal configuration file [PATH] produced an unexpected error: expected [PATH] to be a symlink. Skipping.\",\"Renewal configuration file [PATH] produced an unexpected error: expected [PATH] to be a symlink. Skipping.\",\"Renewal configuration file [PATH] produced an unexpected error: expected [PATH] to be a symlink. Skipping.\",\"Renewal configuration file [PATH] produced an unexpected error: expected [PATH] to be a symlink. Skipping.\",\"The following renewal configurations were invalid:\",\" [PATH]\",\" [PATH]\",\" [PATH]\",\" [PATH]\",\" [PATH]\",\" [PATH]\",\" [PATH]\",\" [PATH]\",\" [PATH]\",\" [PATH]\",\" [PATH]\",\" [PATH]\",\" [PATH]\"],\"certificates_stderr_sha\":\"d744a29aaabe2413\",\"show_account_rc\":1,\"show_account_diag\":[\"An unexpected error occurred:\",\"ValueError: Requesting acme-v02.api.letsencrypt.org/directory: No route to host\"],\"gitread_live_exists\":false,\"gitread_renewal_exists\":false,\"new_vhost_files_exist\":[]},\"locks\":{\"present\":[]},\"processes\":{\"certbot_related\":[]}}}}\nTASK_COMPLETE=true\nTASK_RESULT=PASS_SPB_ACME_CERTBOT_FAIL_RCA\nDECISION=READ_EXACT_9124_THEN_CORRECT_ACME_EXECUTION_PATH\n" } diff --git a/runtime/latest.txt b/runtime/latest.txt index 9d6c8458..7e4e1aa1 100644 --- a/runtime/latest.txt +++ b/runtime/latest.txt @@ -1,15 +1,15 @@ CHAT_OUTPUT_BEGIN -COMMAND_ID=NEWFI-260922-NEWFDOM-DR221-VM210-PRODUCER-MEMBERSHIP-RCA416R1 +COMMAND_ID=SUPPORT-260922-SPB-ACME-CERTBOT-FAIL-RCA-9124R1 STATUS=OK RC=0 HOST=pve01 MODE=read-only -COMPONENT=newfi-dr221-vm210-producer-membership-rca +COMPONENT=spb-acme-certbot-fail-rca REFERENCE_REGISTER_CHECK=OK REFERENCE_SHA256=5a3d8e5154c41cb582a4e0aca68090be1f0138918bf82131a948df326f9d8d66 ERROR_REGISTER_CHECK=OK ERROR_REGISTER_SHA256=3b09a553ec0f527ed3afeed4753f52a74ee3036045bbfb3c685e9f8af4ba7ba0 -COMMAND_SHA256=f147173d66cd34c3f641e8e304fe06d95889d3fb0886845c883d37b7494f102f +COMMAND_SHA256=65e45035e738859d7301cc86383acdbc514310a22e8a10f9a86928f593b10521 DUPLICATE_FAILED_COMMAND_BLOCKED=false EXECUTION_STARTED=true CHANGE_DECLARED=false @@ -24,193 +24,13 @@ MUTATION_OUTCOME=NO_MUTATION SANITIZED=yes SECRETS_INCLUDED=no PRIVATE_ADDRESSES_INCLUDED=no -RAW_EVIDENCE_SHA256=df560dcc7c495f7172790e5839527d5a14299e541ad404e0fee0a82517a452eb -SANITIZED_OUTPUT_SHA256=4c78ca1b8648c6cbd612ae432ff927d18627ba6573375c6afc3af25de784b531 +RAW_EVIDENCE_SHA256=0d656c5cf4c2e5febffda0117d173fb22f193ad50645d09c55e3eb5d4bbcb85c +SANITIZED_OUTPUT_SHA256=0d656c5cf4c2e5febffda0117d173fb22f193ad50645d09c55e3eb5d4bbcb85c OUTPUT_BEGIN -NEWFI416_BEGIN=true -COMMAND_ID=NEWFI-260922-NEWFDOM-DR221-VM210-PRODUCER-MEMBERSHIP-RCA416R1 -MODE=read-only -COMPONENT=newfi-dr221-vm210-producer-membership-rca -SCOPE=PROVE_OR_REFUTE_VM210_MEMBERSHIP_IN_CANONICAL_VM_CLOUD_QUORUM_PRODUCER -MUTATION_BOUNDARY=NONE;NO_BACKUP;NO_DISPATCH;NO_SYSTEMD_ACTION;NO_CLOUD_WRITE;NO_VM_WRITE;NO_GIT_WRITE -REFERENCE_SHA256=5a3d8e5154c41cb582a4e0aca68090be1f0138918bf82131a948df326f9d8d66 -ERROR_REGISTER_SHA256=3b09a553ec0f527ed3afeed4753f52a74ee3036045bbfb3c685e9f8af4ba7ba0 -RUNNER_SHA256=b248a4c32c9cc64e5747e7dce6c7fc0a23f5124a77c71ce72e27a81aceae9d2d -QUEUE_SHA256=fbc27ad67fc3d3f51934e1d73180fe0df8c9d15f129f34ebba39c56aa1d03612 -FRESHNESS_SHA256=bd6257397c6ea910df2ea0dd57252b33bccd0bbd965175840486dfe5e079bf96 -NODE_WORKER_SHA256=91bdd116be865560a38105dffbd1a66d687232035e2359e4a5be4d225e9c61d1 -VM210_ALL_TARGET_ROWS=2 -VM210_PVE_TARGET_COUNT=1 -VM210_XF_TARGET_COUNT=1 -VM210_PVE_ENABLED=true -VM210_XF_ENABLED=true -PRODUCER_MEMBERSHIP_SOURCE_BEGIN=true -PRODUCER_FILE=/usr/local/sbin/homelab-vm-cloud-quorum-queue|SHA256=fbc27ad67fc3d3f51934e1d73180fe0df8c9d15f129f34ebba39c56aa1d03612 -9-FRESHNESS_SECONDS=72000 -10-DUE_SECONDS=43200 -11-SSH_BASE=(-o BatchMode=yes -o ConnectTimeout=12 -o StrictHostKeyChecking=yes -o UserKnownHostsFile=[SENSITIVE_PATH] -12:ALL_VMIDS=(160 170 130 171 180 190 150 9130 110 111 112 113 200) -13-MIGRATE_VMIDS=(160 170 130 171 180 190 150 9130 111 112 113) -14- -15-die(){ printf 'VM_QUORUM_QUEUE_ERROR=%s\n' "$1" >&2; exit "${2:-65}"; } -16-(( DUE_SECONDS > 0 && DUE_SECONDS < FRESHNESS_SECONDS )) || die DUE_THRESHOLD_INVALID 65 -17:expected_owner(){ -18- case "$1" in -19- 110|112|150|170|171) printf 'pve01\n' ;; -20- 111|113|160|180|9130) printf 'pve02\n' ;; -21- 130|190|200) printf 'pve03\n' ;; -22: *) die "VMID_NOT_ALLOWED_$1" 64 ;; -23- esac -24-} -25-observed_owner(){ --- -45-} -46-_run_worker_unlocked() { -47- local vmid="$1" action="$2" generation="${3:-}" owner observed host -48: owner="$(expected_owner "$vmid")" -49- observed="$(observed_owner "$vmid")" -50- [[ "$observed" == "$owner" ]] || die "CLUSTER_OWNER_MISMATCH_${vmid}_${observed}_EXPECTED_${owner}" 65 -51- if [[ "$owner" == "pve01" ]]; then --- -61-} -62-run_probe(){ -63- local vmid="$1" owner observed host -64: owner="$(expected_owner "$vmid")" -65- observed="$(observed_owner "$vmid")" -66- [[ "$observed" == "$owner" ]] || die "CLUSTER_OWNER_MISMATCH_${vmid}_${observed}_EXPECTED_${owner}" 65 -67- if [[ "$owner" == "pve01" ]]; then --- -73-} -74-plan_all(){ -75- local vmid -76: for vmid in "${ALL_VMIDS[@]}"; do -77: printf 'QUEUE_PLAN VMID=%s EXPECTED_OWNER=%s OBSERVED_OWNER=%s\n' "$vmid" "$(expected_owner "$vmid")" "$(observed_owner "$vmid")" -78- done -79-} -80-freshness_decision(){ --- -116-freshness_plan(){ -117- local vmid now_epoch out decision -118- [[ -x "$PROBE" ]] || die "FRESHNESS_PROBE_NOT_INSTALLED" 69 -119: for vmid in "${ALL_VMIDS[@]}"; do -120- now_epoch="$(date -u +%s)" -121- out="$(freshness_decision "$vmid" "$now_epoch")" -122- printf '%s\n' "$out" --- -132- exec 8>"$HEAVY_LOCK" -133- flock -n 8 || die "GLOBAL_HEAVY_LOCK_BUSY" 75 -134- for vmid in "$@"; do -135: printf 'QUEUE_BEGIN VMID=%s OWNER=%s MODE=FORCE\n' "$vmid" "$(expected_owner "$vmid")" -136- out="$(exec 8>&-; exec 9>&-; run_worker "$vmid" backup)" -137- printf '%s\n' "$out" -138- generation="$(awk -F= '$1=="GENERATION"{print $2}' <<<"$out" | tail -1)" --- -146-dispatch_one(){ -147- local vmid="$1" freshness decision now_epoch out generation -148- [[ "${HOMELAB_BACKUP_DISPATCHER:-}" == 1 ]] || die "DISPATCH_ONE_DIRECT_FORBIDDEN" 64 -149: expected_owner "$vmid" >/dev/null -150- exec 9>"$LOCK" -151- if ! flock -n 9; then -152- printf 'QUEUE_DEFERRED KIND=VM REASON=GLOBAL_QUEUE_LOCK_BUSY VMID=%s\n' "$vmid" --- -170- BACKUP_RESUME|BACKUP_STALE|BACKUP_DUE|BACKUP_UNVERIFIED) ;; -171- *) die "FRESHNESS_RECHECK_INVALID_${vmid}_${decision:-EMPTY}" 65 ;; -172- esac -173: printf 'QUEUE_BEGIN VMID=%s OWNER=%s MODE=%s DISPATCH=ONE_TARGET\n' "$vmid" "$(expected_owner "$vmid")" "$decision" -174- out="$(exec 8>&-; exec 9>&-; run_worker "$vmid" backup)" -175- printf '%s\n' "$out" -176- generation="$(awk -F= '$1=="GENERATION"{print $2}' <<<"$out" | tail -1)" -PRODUCER_FILE=/usr/local/libexec/homelab-vm-cloud-quorum-freshness-probe|SHA256=bd6257397c6ea910df2ea0dd57252b33bccd0bbd965175840486dfe5e079bf96 -5-RESTORE_ROOT=/var/lib/homelab-backup/vm-restore-validation -6-HOST="$(hostname -s)" -7-die(){ printf 'VM_FRESHNESS_PROBE_ERROR=%s\n' "$1" >&2; exit "${2:-65}"; } -8:expected_owner(){ -9-case "$1" in -10-110|112|150|170|171) printf 'pve01\n' ;; -11-111|113|160|180|9130) printf 'pve02\n' ;; -12-130|190|200) printf 'pve03\n' ;; -13:*) die "VMID_NOT_ALLOWED_$1" 64 ;; -14-esac -15-} -16-assert_local_identity(){ -17-local vmid="$1" owner -18:owner="$(expected_owner "$vmid")" -19-[[ "$HOST" == "$owner" ]] || die "OWNER_MISMATCH_${vmid}_${HOST}_EXPECTED_${owner}" 65 -20-} -21-state_scan(){ -PRODUCER_FILE=/usr/local/libexec/homelab-vm-cloud-quorum-node-worker|SHA256=91bdd116be865560a38105dffbd1a66d687232035e2359e4a5be4d225e9c61d1 -21-active_backup_restore(){ local rc; if pgrep -af '(^|/)(vzdump|qmrestore|pctrestore|vma|zstd|rclone)([[:space:]]|$)'; then return 0; else rc=$?; [[ "$rc" == 1 ]] && return 0; die "PGREP_FAILED_$rc" 69; fi; } -22-snapshot_count_160(){ qm listsnapshot 160 | awk '/->/ && $0 !~ /current/{n++} END{print n+0}'; } -23-vm_lock(){ qm config "$1" | awk -F': ' '$1=="lock"{print $2; found=1} END{if(!found) print "none"}'; } -24:expected_owner(){ -25-case "$1" in -26-110|112|150|170|171) printf 'pve01\n' ;; -27-111|113|160|180|9130) printf 'pve02\n' ;; -28-130|190|200) printf 'pve03\n' ;; -29:*) die "VMID_NOT_ALLOWED_$1" 64 ;; -30-esac -31-} -32:expected_type(){ -33-case "$1" in -34:110|111|112|113|200) printf 'lxc\n' ;; -35:130|150|160|170|171|180|190|9130) printf 'qemu\n' ;; -36:*) die "VMID_NOT_ALLOWED_$1" 64 ;; -37-esac -38-} -39-expected_status(){ --- -45-} -46-assert_local_identity(){ -47-local vmid="$1" owner type status expect_status -48:owner="$(expected_owner "$vmid")" -49:type="$(expected_type "$vmid")" -50-[[ "$HOST" == "$owner" ]] || die "OWNER_MISMATCH_${vmid}_${HOST}_EXPECTED_${owner}" 65 -51-if [[ "$type" == "qemu" ]]; then -52-[[ -s "/etc/pve/qemu-server/${vmid}.conf" ]] || die "QEMU_CONFIG_MISSING_$vmid" 66 --- -153-plan(){ -154-local vmid="$1" owner type status -155-assert_local_identity "$vmid" -156:owner="$(expected_owner "$vmid")" -157:type="$(expected_type "$vmid")" -158-status="$(expected_status "$vmid")" -159-printf 'PLAN_STATUS=OK\nVMID=%s\nOWNER=%s\nTYPE=%s\nEXPECTED_STATUS=%s\n' "$vmid" "$owner" "$type" "$status" -160-} -PRODUCER_MEMBERSHIP_SOURCE_END=true -VM210_FRESHNESS_DIRECT_RC=64 -VM210_FRESHNESS_STDOUT_SHA256=e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855 -VM210_FRESHNESS_STDERR_SHA256=b0d9e4d7ad61d749dfaf399ab3ad606613113a88a2c3e7085ff17e2463dc8b9e -VM210_FRESHNESS_NOT_ALLOWED=true -QUEUE_PLAN_RC=0 -QUEUE_PLAN_HAS_VM210=false -QUEUE_PLAN_STDERR_SHA256=e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855 -STATIC_FILE=/usr/local/sbin/homelab-vm-cloud-quorum-queue -STATIC_CONTAINS_210_LITERAL=false -STATIC_CONTAINS_VM9130_LITERAL=true -STATIC_LINE=12:ALL_VMIDS=(160 170 130 171 180 190 150 9130 110 111 112 113 200) -STATIC_LINE=22: *) die "VMID_NOT_ALLOWED_$1" 64 ;; -STATIC_LINE=76: for vmid in "${ALL_VMIDS[@]}"; do -STATIC_LINE=119: for vmid in "${ALL_VMIDS[@]}"; do -STATIC_FILE=/usr/local/libexec/homelab-vm-cloud-quorum-freshness-probe -STATIC_CONTAINS_210_LITERAL=false -STATIC_CONTAINS_VM9130_LITERAL=true -STATIC_LINE=13:*) die "VMID_NOT_ALLOWED_$1" 64 ;; -STATIC_FILE=/usr/local/libexec/homelab-vm-cloud-quorum-node-worker -STATIC_CONTAINS_210_LITERAL=false -STATIC_CONTAINS_VM9130_LITERAL=true -STATIC_LINE=29:*) die "VMID_NOT_ALLOWED_$1" 64 ;; -STATIC_LINE=34:110|111|112|113|200) printf 'lxc\n' ;; -STATIC_LINE=35:130|150|160|170|171|180|190|9130) printf 'qemu\n' ;; -STATIC_LINE=36:*) die "VMID_NOT_ALLOWED_$1" 64 ;; -OWNER_STATE|service=activating|sub=start|heavy_lock=BUSY -DECISION=PASS_DR221_416_VM210_PRODUCER_MEMBERSHIP_MISSING_PROVEN -NEXT_GATE=WAIT_OWNER_THEN_REPAIR_VM210_CLOUD_QUORUM_PRODUCER_MEMBERSHIP_AND_FINAL_DR -CHANGES_MADE_BY_416=false -PRODUCT_MUTATION_BY_416=false -VM_MUTATION_BY_416=false -CLOUD_MUTATION_BY_416=false -HOMELAB_RESULT_CONTRACT={"version":1,"command_id":"NEWFI-260922-NEWFDOM-DR221-VM210-PRODUCER-MEMBERSHIP-RCA416R1","status":"OK","changes_made":false,"rollback_started":false,"rollback_restored":null} -NEWFI416_END=true +SPB9124_RCA={"guest":{"ok":true,"data":{"hook":{"exists":true,"sha256":"dcaffc5a248359fa3ed93b7bf8af33ad69985776c4a8e3e7f2016c6168da5aee","sha_matches_9120":true,"functions":["out","token","api","zone_id","state_path","auth","cleanup"],"safe_to_import_without_main":false,"unsafe_top_level_types":["Expr"],"selected_string_constants":[".json","/dns_records","/dns_records/","/opt/npmplus/secure/gram1_cf_token","/zones/","/zones?","GRAM1_CF_ZONE_LOOKUP ok=no http=","GRAM1_CF_ZONE_LOOKUP ok=yes id_prefix=","_acme-challenge.","gram1.ru","https://api.cloudflare.com/client/v4","www.gram1.ru","zone_id"],"function_sources":{"token":["def token(): return TOKEN_PATH.read_text(encoding=\"utf-8\").strip()"],"api":["def api(method, path, payload=None):"," data = json.dumps(payload).encode(\"utf-8\") if payload is not None else None"," req = urllib.request.Request(\"https://api.cloudflare.com/client/v4\" + path, data=data, headers={\"Authorization\": \"Bearer[REDACTED_EXPR]\" + token(), \"Content-Type\": \"application/json\"}, method=method)"," try:"," with urllib.request.urlopen(req, timeout=25) as r: return r.status, json.loads(r.read().decode(\"utf-8\", \"replace\"))"," except urllib.error.HTTPError as e:"," try: body = json.loads(e.read().decode(\"utf-8\", \"replace\"))"," except Exception: body = {\"success\": False, \"errors\": [{\"message\": \"http_error_non_json\"}]}"," return e.code, body"],"zone_id":["def zone_id():"," status, data = api(\"GET\", \"/zones?\" + urllib.parse.urlencode({\"name\": ZONE}))"," if not data.get(\"success\") or not data.get(\"result\"):"," out(f\"GRAM1_CF_ZONE_LOOKUP ok=no http={status}\"); return None"," zid = data[\"result\"][0][\"id\"]; out(f\"GRAM1_CF_ZONE_LOOKUP ok=yes id_prefix={zid[:8]}\"); return zid"],"auth":["def auth():"," domain = os.environ.get(\"CERTBOT_DOMAIN\", \"\"); validation = os.environ.get(\"CERTBOT_VALIDATION\", \"\")"," if domain not in (\"gram1.ru\", \"www.gram1.ru\") or not validation:"," out(f\"GRAM1_CF_AUTH domain={domain} status=review reason=bad_env\"); return 2"," zid = zone_id()"," if not zid: return 2"," name = \"_acme-challenge.\" + domain"," status, data = api(\"POST\", f\"/zones/{zid}/dns_records\", {\"type\": \"TXT\", \"name\": name, \"content\": validation, \"ttl\": 120})"," if not data.get(\"success\"):"," errs = data.get(\"errors\") or []; msg = str(errs[0].get(\"message\", \"\"))[:80].replace(\"\\n\", \" \") if errs else \"\""," out(f\"GRAM1_CF_AUTH domain={domain} status=review http={status} error={msg}\"); return 2"," rec = data[\"result\"][\"id\"]; state_path(domain, validation).write_text(json.dumps({\"zone_id\": zid, \"record_id\": rec}), encoding=\"utf-8\")"," out(f\"GRAM1_CF_AUTH domain={domain} status=ok record_id_prefix={rec[:8]}\"); time.sleep(45); return 0"],"cleanup":["def cleanup():"," domain = os.environ.get(\"CERTBOT_DOMAIN\", \"\"); validation = os.environ.get(\"CERTBOT_VALIDATION\", \"\")"," sp = state_path(domain, validation)"," if sp.exists():"," try:"," data = json.loads(sp.read_text(encoding=\"utf-8\"))"," status, resp = api(\"DELETE\", f\"/zones/{data['zone_id']}/dns_records/{data['record_id']}\")"," out(f\"GRAM1_CF_CLEANUP domain={domain} status={'ok' if resp.get('success') else 'review'} http={status}\"); sp.unlink(missing_ok=True)"," except Exception as e: out(f\"GRAM1_CF_CLEANUP domain={domain} status=review error={type(e).__name__}\")"," else: out(f\"GRAM1_CF_CLEANUP domain={domain} status=skip reason=no_state\")"," return 0"]}},"cloudflare_readonly":{},"certbot":{"data_mount_found":true,"data_mount_hash":"ec0ac67820f88633","config_exists":true,"config_mode":"0o700","account_json_count":6,"renewal_count":13,"renewal_basenames":["hbarhub.ru.conf","newfi-staging-gram1-ru.conf","newfilya.ru.conf","npm-17.conf","npm-21.conf","npm-26.conf","npm-27.conf","npm-29.conf","npm-30.conf","npm-31.conf","npm-32.conf","npm-35.conf","npm-38.conf"],"certificates_rc":0,"certificates_diag":["Renewal configuration file [PATH] produced an unexpected error: expected [PATH] to be a symlink. Skipping.","Renewal configuration file [PATH] produced an unexpected error: expected [PATH] to be a symlink. Skipping.","Renewal configuration file [PATH] produced an unexpected error: expected [PATH] to be a symlink. Skipping.","Renewal configuration file [PATH] produced an unexpected error: expected [PATH] to be a symlink. Skipping.","Renewal configuration file [PATH] produced an unexpected error: expected [PATH] to be a symlink. Skipping.","Renewal configuration file [PATH] produced an unexpected error: expected [PATH] to be a symlink. Skipping.","Renewal configuration file [PATH] produced an unexpected error: expected [PATH] to be a symlink. Skipping.","Renewal configuration file [PATH] produced an unexpected error: expected [PATH] to be a symlink. Skipping.","Renewal configuration file [PATH] produced an unexpected error: expected [PATH] to be a symlink. Skipping.","Renewal configuration file [PATH] produced an unexpected error: expected [PATH] to be a symlink. Skipping.","Renewal configuration file [PATH] produced an unexpected error: expected [PATH] to be a symlink. Skipping.","Renewal configuration file [PATH] produced an unexpected error: expected [PATH] to be a symlink. Skipping.","Renewal configuration file [PATH] produced an unexpected error: expected [PATH] to be a symlink. Skipping.","The following renewal configurations were invalid:"," [PATH]"," [PATH]"," [PATH]"," [PATH]"," [PATH]"," [PATH]"," [PATH]"," [PATH]"," [PATH]"," [PATH]"," [PATH]"," [PATH]"," [PATH]"],"certificates_stderr_sha":"d744a29aaabe2413","show_account_rc":1,"show_account_diag":["An unexpected error occurred:","ValueError: Requesting acme-v02.api.letsencrypt.org/directory: No route to host"],"gitread_live_exists":false,"gitread_renewal_exists":false,"new_vhost_files_exist":[]},"locks":{"present":[]},"processes":{"certbot_related":[]}}}} +TASK_COMPLETE=true +TASK_RESULT=PASS_SPB_ACME_CERTBOT_FAIL_RCA +DECISION=READ_EXACT_9124_THEN_CORRECT_ACME_EXECUTION_PATH OUTPUT_END CHAT_OUTPUT_END