diff --git a/runtime/history/SUPPORT-260924-BEGET-DNS-D01-CLUSTER-COVERAGE-9166R1.json b/runtime/history/SUPPORT-260924-BEGET-DNS-D01-CLUSTER-COVERAGE-9166R1.json new file mode 100644 index 00000000..4d6012fb --- /dev/null +++ b/runtime/history/SUPPORT-260924-BEGET-DNS-D01-CLUSTER-COVERAGE-9166R1.json @@ -0,0 +1,38 @@ +{ + "schema_version": 1, + "channel": "homelab-runtime", + "command_id": "SUPPORT-260924-BEGET-DNS-D01-CLUSTER-COVERAGE-9166R1", + "status": "FAIL", + "rc": 42, + "host": "pve01", + "mode": "read-only", + "component": "beget-dns-d01-cloudflare-inventory", + "started_at_utc": "2026-09-24T16:48:34Z", + "finished_at_utc": "2026-09-24T16:49:58Z", + "reference_register_checked": true, + "reference_sha256": "5a3d8e5154c41cb582a4e0aca68090be1f0138918bf82131a948df326f9d8d66", + "error_register_checked": true, + "error_register_sha256": "3b09a553ec0f527ed3afeed4753f52a74ee3036045bbfb3c685e9f8af4ba7ba0", + "command_sha256": "3fdfe7b2879eee2cf217828c2f81f14d30158e549f0fdddb53c1f9dbe1ff074f", + "duplicate_failed_command_blocked": false, + "block_reason": null, + "execution_started": true, + "change_declared": false, + "result_contract_valid": true, + "result_contract_status": null, + "result_contract_error": null, + "command_rc": 42, + "changes_made": false, + "rollback_started": false, + "rollback_restored": null, + "mutation_outcome": "NO_MUTATION", + "sanitized": true, + "secrets_included": false, + "private_addresses_included": false, + "raw_evidence_retained_locally": true, + "raw_evidence_sha256": "5a92c486b9fa5324f80d8a7592d804b03f89ac043571bf303eb0b376567273c6", + "sanitized_output_sha256": "602d96b59e5b34e3ca818c05c9e047d90ecca7db5a081311cabafddd99f0a241", + "output_truncated_in_json": false, + "full_sanitized_output_url": "https://git.gram1.ru/.well-known/homelab-runtime/latest.txt", + "output": "DNSD01_9166_BEGIN=true\nRCA_FIX=EXPAND_CLOUDFLARE_AUTH_DISCOVERY_ACROSS_PVE_NODES_RUNNING_LXC_QEMU_AND_EDGE\nMUTATION_BOUNDARY=READ_ONLY_CLOUDFLARE_AND_PUBLIC_DNS_INVENTORY;NO_DNS_RECORD_NS_DS_OR_SITE_CHANGE\nPREVIOUS_EXACT_9165=PASS\nFRESH_ASSISTANT_RUNTIME_HEAD=726e443a1d83c2009482e32ca0301f603ea60f7f\nD01_SOURCE_PVE01_RC=0\nD01_SOURCE_PVE01_LINE_COUNT=6\nD01_SOURCE_PVE02_RC=0\nD01_SOURCE_PVE02_LINE_COUNT=1\nD01_SOURCE_PVE03_RC=0\nD01_SOURCE_PVE03_LINE_COUNT=5\nD01_SOURCE_EDGE_RC=0\nD01_SOURCE_EDGE_LINE_COUNT=3\nD01_CLUSTER_AUTH_RAW_LINE_COUNT=15\nD01_CF_AUTH_DISCOVERY=PASS\nD01_CF_AUTH_CANDIDATE_COUNT=4\nD01_AUTH_1_KIND=BEARER\n[REDACTED]\nD01_AUTH_1_DNS_RECORD_READ_ZONE_COUNT=0\nD01_AUTH_1_DNSSEC_READ_ZONE_COUNT=0\nD01_AUTH_2_KIND=BEARER\n[REDACTED]\nD01_AUTH_2_DNS_RECORD_READ_ZONE_COUNT=1\nD01_AUTH_2_DNSSEC_READ_ZONE_COUNT=1\nD01_AUTH_3_KIND=BEARER\n[REDACTED]\nD01_AUTH_3_DNS_RECORD_READ_ZONE_COUNT=1\nD01_AUTH_3_DNSSEC_READ_ZONE_COUNT=1\nD01_AUTH_4_KIND=BEARER\n[REDACTED]\nD01_AUTH_4_DNS_RECORD_READ_ZONE_COUNT=5\nD01_AUTH_4_DNSSEC_READ_ZONE_COUNT=5\nD01_CF_ACCESSIBLE_ZONE_COUNT=13\nD01_EXPECTED_ZONE_COUNT=13\nD01_EXPECTED_MISSING_COUNT=0\nD01_EXPECTED_MISSING=NONE\nD01_EXTRA_ZONE_COUNT=0\nD01_EXTRA_ZONES=NONE\nD01_ZONESET_SHA256=766204ad5d819bfa7cf96cc42b8fbde659529b923b9b4aaa32c1c83d1bd9649e\nD01_DNS_RECORD_READABLE_ZONE_COUNT=7\nD01_DNS_RECORD_UNREADABLE_ZONE_COUNT=6\nD01_DNS_RECORD_UNREADABLE_ZONES=hapusya.ru,hbarhub.ru,kingofwolk.ru,newfi.ru,newfilya.ru,pvepro.ru\nD01_EXPECTED_DNS_UNREADABLE_COUNT=6\nD01_EXPECTED_DNS_UNREADABLE=hapusya.ru,hbarhub.ru,kingofwolk.ru,newfi.ru,newfilya.ru,pvepro.ru\nD01_COMPLETENESS=INCOMPLETE_CLOUDFLARE_SCOPE_OR_DNS_READ\nTASK_COMPLETE=true\nTASK_RESULT=FAIL_DNS_D01_CLUSTER_COVERAGE\nDECISION=D01_DNS_RECORD_READ_SCOPE_STILL_INCOMPLETE\nNEXT_GATE=READ_EXACT_9166_AND_FIX_ONLY_D01_PROVEN_BLOCKER\nHOMELAB_RESULT_CONTRACT={\"version\":1,\"command_id\":\"SUPPORT-260924-BEGET-DNS-D01-CLUSTER-COVERAGE-9166R1\",\"status\":\"FAIL\",\"changes_made\":false,\"rollback_started\":false,\"rollback_restored\":null}\n" +} diff --git a/runtime/history/SUPPORT-260924-BEGET-DNS-D01-CLUSTER-COVERAGE-9166R1.txt b/runtime/history/SUPPORT-260924-BEGET-DNS-D01-CLUSTER-COVERAGE-9166R1.txt new file mode 100644 index 00000000..04c8f9e9 --- /dev/null +++ b/runtime/history/SUPPORT-260924-BEGET-DNS-D01-CLUSTER-COVERAGE-9166R1.txt @@ -0,0 +1,82 @@ +CHAT_OUTPUT_BEGIN +COMMAND_ID=SUPPORT-260924-BEGET-DNS-D01-CLUSTER-COVERAGE-9166R1 +STATUS=FAIL +RC=42 +HOST=pve01 +MODE=read-only +COMPONENT=beget-dns-d01-cloudflare-inventory +REFERENCE_REGISTER_CHECK=OK +REFERENCE_SHA256=5a3d8e5154c41cb582a4e0aca68090be1f0138918bf82131a948df326f9d8d66 +ERROR_REGISTER_CHECK=OK +ERROR_REGISTER_SHA256=3b09a553ec0f527ed3afeed4753f52a74ee3036045bbfb3c685e9f8af4ba7ba0 +COMMAND_SHA256=3fdfe7b2879eee2cf217828c2f81f14d30158e549f0fdddb53c1f9dbe1ff074f +DUPLICATE_FAILED_COMMAND_BLOCKED=false +EXECUTION_STARTED=true +CHANGE_DECLARED=false +RESULT_CONTRACT_VALID=true +RESULT_CONTRACT_STATUS=NOT_APPLICABLE +RESULT_CONTRACT_ERROR=NONE +COMMAND_RC=42 +CHANGES_MADE=false +ROLLBACK_STARTED=false +ROLLBACK_RESTORED=null +MUTATION_OUTCOME=NO_MUTATION +SANITIZED=yes +SECRETS_INCLUDED=no +PRIVATE_ADDRESSES_INCLUDED=no +RAW_EVIDENCE_SHA256=5a92c486b9fa5324f80d8a7592d804b03f89ac043571bf303eb0b376567273c6 +SANITIZED_OUTPUT_SHA256=602d96b59e5b34e3ca818c05c9e047d90ecca7db5a081311cabafddd99f0a241 +OUTPUT_BEGIN +DNSD01_9166_BEGIN=true +RCA_FIX=EXPAND_CLOUDFLARE_AUTH_DISCOVERY_ACROSS_PVE_NODES_RUNNING_LXC_QEMU_AND_EDGE +MUTATION_BOUNDARY=READ_ONLY_CLOUDFLARE_AND_PUBLIC_DNS_INVENTORY;NO_DNS_RECORD_NS_DS_OR_SITE_CHANGE +PREVIOUS_EXACT_9165=PASS +FRESH_ASSISTANT_RUNTIME_HEAD=726e443a1d83c2009482e32ca0301f603ea60f7f +D01_SOURCE_PVE01_RC=0 +D01_SOURCE_PVE01_LINE_COUNT=6 +D01_SOURCE_PVE02_RC=0 +D01_SOURCE_PVE02_LINE_COUNT=1 +D01_SOURCE_PVE03_RC=0 +D01_SOURCE_PVE03_LINE_COUNT=5 +D01_SOURCE_EDGE_RC=0 +D01_SOURCE_EDGE_LINE_COUNT=3 +D01_CLUSTER_AUTH_RAW_LINE_COUNT=15 +D01_CF_AUTH_DISCOVERY=PASS +D01_CF_AUTH_CANDIDATE_COUNT=4 +D01_AUTH_1_KIND=BEARER +[REDACTED] +D01_AUTH_1_DNS_RECORD_READ_ZONE_COUNT=0 +D01_AUTH_1_DNSSEC_READ_ZONE_COUNT=0 +D01_AUTH_2_KIND=BEARER +[REDACTED] +D01_AUTH_2_DNS_RECORD_READ_ZONE_COUNT=1 +D01_AUTH_2_DNSSEC_READ_ZONE_COUNT=1 +D01_AUTH_3_KIND=BEARER +[REDACTED] +D01_AUTH_3_DNS_RECORD_READ_ZONE_COUNT=1 +D01_AUTH_3_DNSSEC_READ_ZONE_COUNT=1 +D01_AUTH_4_KIND=BEARER +[REDACTED] +D01_AUTH_4_DNS_RECORD_READ_ZONE_COUNT=5 +D01_AUTH_4_DNSSEC_READ_ZONE_COUNT=5 +D01_CF_ACCESSIBLE_ZONE_COUNT=13 +D01_EXPECTED_ZONE_COUNT=13 +D01_EXPECTED_MISSING_COUNT=0 +D01_EXPECTED_MISSING=NONE +D01_EXTRA_ZONE_COUNT=0 +D01_EXTRA_ZONES=NONE +D01_ZONESET_SHA256=766204ad5d819bfa7cf96cc42b8fbde659529b923b9b4aaa32c1c83d1bd9649e +D01_DNS_RECORD_READABLE_ZONE_COUNT=7 +D01_DNS_RECORD_UNREADABLE_ZONE_COUNT=6 +D01_DNS_RECORD_UNREADABLE_ZONES=hapusya.ru,hbarhub.ru,kingofwolk.ru,newfi.ru,newfilya.ru,pvepro.ru +D01_EXPECTED_DNS_UNREADABLE_COUNT=6 +D01_EXPECTED_DNS_UNREADABLE=hapusya.ru,hbarhub.ru,kingofwolk.ru,newfi.ru,newfilya.ru,pvepro.ru +D01_COMPLETENESS=INCOMPLETE_CLOUDFLARE_SCOPE_OR_DNS_READ +TASK_COMPLETE=true +TASK_RESULT=FAIL_DNS_D01_CLUSTER_COVERAGE +DECISION=D01_DNS_RECORD_READ_SCOPE_STILL_INCOMPLETE +NEXT_GATE=READ_EXACT_9166_AND_FIX_ONLY_D01_PROVEN_BLOCKER +HOMELAB_RESULT_CONTRACT={"version":1,"command_id":"SUPPORT-260924-BEGET-DNS-D01-CLUSTER-COVERAGE-9166R1","status":"FAIL","changes_made":false,"rollback_started":false,"rollback_restored":null} + +OUTPUT_END +CHAT_OUTPUT_END diff --git a/runtime/latest.json b/runtime/latest.json index fee08416..4d6012fb 100644 --- a/runtime/latest.json +++ b/runtime/latest.json @@ -1,19 +1,19 @@ { "schema_version": 1, "channel": "homelab-runtime", - "command_id": "SUPPORT-260924-SKLADCHIK-MONITOR-STATUS-ERROR-RCA-73167R1", - "status": "OK", - "rc": 0, + "command_id": "SUPPORT-260924-BEGET-DNS-D01-CLUSTER-COVERAGE-9166R1", + "status": "FAIL", + "rc": 42, "host": "pve01", "mode": "read-only", - "component": "skladchik-monitor-status-error-rca", - "started_at_utc": "2026-09-24T16:44:58Z", - "finished_at_utc": "2026-09-24T16:45:00Z", + "component": "beget-dns-d01-cloudflare-inventory", + "started_at_utc": "2026-09-24T16:48:34Z", + "finished_at_utc": "2026-09-24T16:49:58Z", "reference_register_checked": true, "reference_sha256": "5a3d8e5154c41cb582a4e0aca68090be1f0138918bf82131a948df326f9d8d66", "error_register_checked": true, "error_register_sha256": "3b09a553ec0f527ed3afeed4753f52a74ee3036045bbfb3c685e9f8af4ba7ba0", - "command_sha256": "6f2291c89dca3072b8ba3b29c129d594d18708afc9bb806b08dad46f6410d42f", + "command_sha256": "3fdfe7b2879eee2cf217828c2f81f14d30158e549f0fdddb53c1f9dbe1ff074f", "duplicate_failed_command_blocked": false, "block_reason": null, "execution_started": true, @@ -21,7 +21,7 @@ "result_contract_valid": true, "result_contract_status": null, "result_contract_error": null, - "command_rc": 0, + "command_rc": 42, "changes_made": false, "rollback_started": false, "rollback_restored": null, @@ -30,9 +30,9 @@ "secrets_included": false, "private_addresses_included": false, "raw_evidence_retained_locally": true, - "raw_evidence_sha256": "0a9a4ccd3752109628076c7847ace825bed2ce42c2bff682fec4e9e16fd70016", - "sanitized_output_sha256": "0a9a4ccd3752109628076c7847ace825bed2ce42c2bff682fec4e9e16fd70016", + "raw_evidence_sha256": "5a92c486b9fa5324f80d8a7592d804b03f89ac043571bf303eb0b376567273c6", + "sanitized_output_sha256": "602d96b59e5b34e3ca818c05c9e047d90ecca7db5a081311cabafddd99f0a241", "output_truncated_in_json": false, "full_sanitized_output_url": "https://git.gram1.ru/.well-known/homelab-runtime/latest.txt", - "output": "WORKERS2_SKLADCHIK_MONITOR_STATUS_ERROR_RCA_BEGIN=1\nCOMMAND_ID=SUPPORT-260924-SKLADCHIK-MONITOR-STATUS-ERROR-RCA-73167R1\nMODE=read-only\nCOMPONENT=skladchik-monitor-status-error-rca\nMUTATION_BOUNDARY=NONE;READ_ONLY_EDGE_QGA_SERVICE_HEALTH_COOKIE_METADATA_SOURCE_HASH_AND_FILTERED_JOURNAL;TEMP_/run_ONLY;NO_MONITOR_START;NO_SERVICE_ACTION;NO_COOKIE_CONTENT;NO_CONFIG_SECRET_VALUES;NO_NETBIRD_CHANGE;NO_ROUTE_CHANGE;NO_GIT_WRITE\nPREDECESSOR_EXACT=SUPPORT-260924-SKLADCHIK-SUPERVISOR-ROUTE-INVARIANT-REPAIR-73166R1\nPREDECESSOR_DECISION=PASS_ROUTE_INVARIANT_REPAIRED_MONITOR_ERROR_REMAINS\nREFERENCE_SHA256=5a3d8e5154c41cb582a4e0aca68090be1f0138918bf82131a948df326f9d8d66\nERROR_REGISTER_SHA256=3b09a553ec0f527ed3afeed4753f52a74ee3036045bbfb3c685e9f8af4ba7ba0\nRUNNER_SHA256=b248a4c32c9cc64e5747e7dce6c7fc0a23f5124a77c71ce72e27a81aceae9d2d\nEDGE_QGA_TRANSPORT_RC=0\nEDGE_QGA_TRANSPORT_STDERR_SHA256=e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855\nEDGE_QGA_JSON_PARSE=true\nEDGE_QGA_EXITCODE=0\nEDGE_MONITOR_RCA_BEGIN=1\nEDGE_HOSTNAME=edge-vm\nEDGE_TIME_UTC=2026-09-24T16:44:59Z\nskladchik_reports_monitor_service_ACTIVE=failed\nskladchik_reports_monitor_service_ENABLED=static\nskladchik_reports_monitor_service_RESULT=exit-code\nskladchik_reports_monitor_service_EXEC_MAIN_STATUS=1\nskladchik_reports_monitor_timer_ACTIVE=active\nskladchik_reports_monitor_timer_ENABLED=enabled\nskladchik_reports_monitor_timer_RESULT=success\nskladchik_reports_monitor_timer_EXEC_MAIN_STATUS=\nskladchik_reports_monitor_watchdog_timer_ACTIVE=active\nskladchik_reports_monitor_watchdog_timer_ENABLED=enabled\nskladchik_reports_monitor_watchdog_timer_RESULT=success\nskladchik_reports_monitor_watchdog_timer_EXEC_MAIN_STATUS=\nskladchik_reports_monitor_heartbeat_timer_ACTIVE=active\nskladchik_reports_monitor_heartbeat_timer_ENABLED=enabled\nskladchik_reports_monitor_heartbeat_timer_RESULT=success\nskladchik_reports_monitor_heartbeat_timer_EXEC_MAIN_STATUS=\nEDGE_HEALTH_EXISTS=true\nEDGE_HEALTH_SIZE=506\nEDGE_HEALTH_MODE=640\nEDGE_HEALTH_OWNER=0:0\nEDGE_HEALTH_MTIME_EPOCH=1790268286\nEDGE_HEALTH_SHA256=6db70fe9833f5d47a07873cc8c8a2a8ee07e4456b74985efc4c0d69d373ba2f2\nEDGE_ALERT_HEALTH_EXISTS=true\nEDGE_ALERT_HEALTH_SIZE=530\nEDGE_ALERT_HEALTH_MODE=644\nEDGE_ALERT_HEALTH_OWNER=0:0\nEDGE_ALERT_HEALTH_MTIME_EPOCH=1790268142\nEDGE_ALERT_HEALTH_SHA256=d1493c44f34d1aecfae8e35dbee4e4cace3236088921b1a69c786f4c793cc06e\nEDGE_COOKIE_JAR_EXISTS=true\nEDGE_COOKIE_JAR_SIZE=744\nEDGE_COOKIE_JAR_MODE=600\nEDGE_COOKIE_JAR_OWNER=0:0\nEDGE_COOKIE_JAR_MTIME_EPOCH=1790268286\nEDGE_COOKIE_JAR_SHA256=c24634109cab376eab1afe3b3de727d2a7f5b16702ed98f6ee516a3935973e7c\nEDGE_CONFIG_EXISTS=true\nEDGE_CONFIG_SIZE=292\nEDGE_CONFIG_MODE=640\nEDGE_CONFIG_OWNER=0:0\nEDGE_CONFIG_MTIME_EPOCH=1784363565\nEDGE_CONFIG_SHA256=911b8d72a9ca8eb2f4aba14111d5a8acc88898aa0b89de36d7557f0ca10787fd\nEDGE_SECRETS_ENV_EXISTS=true\nEDGE_SECRETS_ENV_SIZE=513\nEDGE_SECRETS_ENV_MODE=600\nEDGE_SECRETS_ENV_OWNER=0:0\nEDGE_SECRETS_ENV_MTIME_EPOCH=1790189249\nEDGE_SECRETS_ENV_SHA256=f2e3757eb9e6f4fd352cf38e4d99ad080d824df5b6916df64dfd036e662ee23b\nEDGE_MONITOR_PY_EXISTS=true\nEDGE_MONITOR_PY_SIZE=25677\nEDGE_MONITOR_PY_MODE=700\nEDGE_MONITOR_PY_OWNER=0:0\nEDGE_MONITOR_PY_MTIME_EPOCH=1786729896\nEDGE_MONITOR_PY_SHA256=bf3a47cf3c53a4efa1f49322f96203a24b9077ea64547162e64c5d3d3326d0ff\nHEALTH_SAFE_BEGIN=1\nHEALTH_KV|STATUS=ERROR\nHEALTH_KV|TS=2026-09-24T16:44:46Z\nHEALTH_KV|error=MonitorError('curl_failed rc=28 stderr=curl: (28) Connection timed out after 15003 milliseconds\\n')\nHEALTH_KV|session_keepalive=persistent_cookie_jar\nHEALTH_KV|cookie_jar=yes\nHEALTH_KV|cookie_jar_bytes=744\nHEALTH_KV|cookie_persistent_count=1\nHEALTH_KV|cookie_expiry_utc=2026-10-23T18:47:24Z\nHEALTH_KV|cookie_expires_in_seconds=2512958\nHEALTH_KV|cookie_expires_in_days=29\nHEALTH_KV|cookie_expiry_status=OK\nHEALTH_KEYS=STATUS,TS,cookie_earliest_expiry_utc,cookie_expires_in_days,cookie_expires_in_seconds,cookie_expiry_status,cookie_expiry_utc,cookie_expiry_warning_days,cookie_jar,cookie_jar_bytes,cookie_jar_mtime,cookie_persistent_count,cookie_session_count,error,session_keepalive\nHEALTH_SAFE_END=1\nALERT_SAFE_BEGIN=1\nALERT_KV|STATUS=ALERT\nALERT_KV|TS=2026-09-24T16:42:22Z\nALERT_KV|monitor_status=ERROR\nALERT_KV|session_keepalive=persistent_cookie_jar\nALERT_KV|cookie_jar=yes\nALERT_KV|cookie_jar_bytes=744\nALERT_KV|cookie_expiry_status=OK\nALERT_KV|cookie_expiry_utc=2026-10-23T18:47:24Z\nALERT_KV|cookie_expires_in_seconds=2513112\nALERT_KV|cookie_expires_in_days=29\nALERT_KEYS=STATUS,TS,cookie_expires_in_days,cookie_expires_in_seconds,cookie_expiry_status,cookie_expiry_utc,cookie_jar,cookie_jar_bytes,health_age_seconds,mail_sent,monitor_open,monitor_reports,monitor_resolved,monitor_status,monitor_timer_active,monitor_timer_enabled,netbird_route_ok,problems,session_keepalive\nALERT_SAFE_END=1\nMONITOR_AST_PARSE_OK=true\nMONITOR_SOURCE_AUTH_MATCH_COUNT=23\nMONITOR_SOURCE_COOKIE_MATCH_COUNT=63\nMONITOR_SOURCE_SESSION_MATCH_COUNT=7\nMONITOR_SOURCE_CAPTCHA_MATCH_COUNT=0\nMONITOR_SOURCE_HTTP_MATCH_COUNT=10\nMONITOR_SOURCE_STATUS_ERROR_MATCH_COUNT=1\nMONITOR_SOURCE_HEALTH_WRITE_MATCH_COUNT=1\n=== MONITOR_RECENT_JOURNAL_FILTERED ===\nMONITOR_JOURNAL|Sep 24 19:29:36 edge-vm systemd[1]: Starting skladchik-reports-monitor.service - Skladchik XenForo reports monitor...\nMONITOR_JOURNAL|Sep 24 19:29:51 edge-vm systemd[1]: skladchik-reports-monitor.service: Main process exited, code=exited, status=1/FAILURE\nMONITOR_JOURNAL|Sep 24 19:29:51 edge-vm systemd[1]: skladchik-reports-monitor.service: Failed with result 'exit-code'.\nMONITOR_JOURNAL|Sep 24 19:29:51 edge-vm systemd[1]: Failed to start skladchik-reports-monitor.service - Skladchik XenForo reports monitor.\nMONITOR_JOURNAL|Sep 24 19:30:07 edge-vm systemd[1]: Starting skladchik-reports-monitor.service - Skladchik XenForo reports monitor...\nMONITOR_JOURNAL|Sep 24 19:30:22 edge-vm systemd[1]: skladchik-reports-monitor.service: Main process exited, code=exited, status=1/FAILURE\nMONITOR_JOURNAL|Sep 24 19:30:22 edge-vm systemd[1]: skladchik-reports-monitor.service: Failed with result 'exit-code'.\nMONITOR_JOURNAL|Sep 24 19:30:22 edge-vm systemd[1]: Failed to start skladchik-reports-monitor.service - Skladchik XenForo reports monitor.\nMONITOR_JOURNAL|Sep 24 19:30:38 edge-vm systemd[1]: Starting skladchik-reports-monitor.service - Skladchik XenForo reports monitor...\nMONITOR_JOURNAL|Sep 24 19:30:53 edge-vm systemd[1]: skladchik-reports-monitor.service: Main process exited, code=exited, status=1/FAILURE\nMONITOR_JOURNAL|Sep 24 19:30:53 edge-vm systemd[1]: skladchik-reports-monitor.service: Failed with result 'exit-code'.\nMONITOR_JOURNAL|Sep 24 19:30:53 edge-vm systemd[1]: Failed to start skladchik-reports-monitor.service - Skladchik XenForo reports monitor.\nMONITOR_JOURNAL|Sep 24 19:31:09 edge-vm systemd[1]: Starting skladchik-reports-monitor.service - Skladchik XenForo reports monitor...\nMONITOR_JOURNAL|Sep 24 19:31:24 edge-vm systemd[1]: skladchik-reports-monitor.service: Main process exited, code=exited, status=1/FAILURE\nMONITOR_JOURNAL|Sep 24 19:31:24 edge-vm systemd[1]: skladchik-reports-monitor.service: Failed with result 'exit-code'.\nMONITOR_JOURNAL|Sep 24 19:31:24 edge-vm systemd[1]: Failed to start skladchik-reports-monitor.service - Skladchik XenForo reports monitor.\nMONITOR_JOURNAL|Sep 24 19:31:40 edge-vm systemd[1]: Starting skladchik-reports-monitor.service - Skladchik XenForo reports monitor...\nMONITOR_JOURNAL|Sep 24 19:31:55 edge-vm systemd[1]: skladchik-reports-monitor.service: Main process exited, code=exited, status=1/FAILURE\nMONITOR_JOURNAL|Sep 24 19:31:55 edge-vm systemd[1]: skladchik-reports-monitor.service: Failed with result 'exit-code'.\nMONITOR_JOURNAL|Sep 24 19:31:55 edge-vm systemd[1]: Failed to start skladchik-reports-monitor.service - Skladchik XenForo reports monitor.\nMONITOR_JOURNAL|Sep 24 19:32:11 edge-vm systemd[1]: Starting skladchik-reports-monitor.service - Skladchik XenForo reports monitor...\nMONITOR_JOURNAL|Sep 24 19:32:26 edge-vm systemd[1]: skladchik-reports-monitor.service: Main process exited, code=exited, status=1/FAILURE\nMONITOR_JOURNAL|Sep 24 19:32:26 edge-vm systemd[1]: skladchik-reports-monitor.service: Failed with result 'exit-code'.\nMONITOR_JOURNAL|Sep 24 19:32:26 edge-vm systemd[1]: Failed to start skladchik-reports-monitor.service - Skladchik XenForo reports monitor.\nMONITOR_JOURNAL|Sep 24 19:32:42 edge-vm systemd[1]: Starting skladchik-reports-monitor.service - Skladchik XenForo reports monitor...\nMONITOR_JOURNAL|Sep 24 19:32:57 edge-vm systemd[1]: skladchik-reports-monitor.service: Main process exited, code=exited, status=1/FAILURE\nMONITOR_JOURNAL|Sep 24 19:32:57 edge-vm systemd[1]: skladchik-reports-monitor.service: Failed with result 'exit-code'.\nMONITOR_JOURNAL|Sep 24 19:32:57 edge-vm systemd[1]: Failed to start skladchik-reports-monitor.service - Skladchik XenForo reports monitor.\nMONITOR_JOURNAL|Sep 24 19:33:12 edge-vm systemd[1]: Starting skladchik-reports-monitor.service - Skladchik XenForo reports monitor...\nMONITOR_JOURNAL|Sep 24 19:33:27 edge-vm systemd[1]: skladchik-reports-monitor.service: Main process exited, code=exited, status=1/FAILURE\nMONITOR_JOURNAL|Sep 24 19:33:27 edge-vm systemd[1]: skladchik-reports-monitor.service: Failed with result 'exit-code'.\nMONITOR_JOURNAL|Sep 24 19:33:27 edge-vm systemd[1]: Failed to start skladchik-reports-monitor.service - Skladchik XenForo reports monitor.\nMONITOR_JOURNAL|Sep 24 19:33:43 edge-vm systemd[1]: Starting skladchik-reports-monitor.service - Skladchik XenForo reports monitor...\nMONITOR_JOURNAL|Sep 24 19:33:58 edge-vm systemd[1]: skladchik-reports-monitor.service: Main process exited, code=exited, status=1/FAILURE\nMONITOR_JOURNAL|Sep 24 19:33:58 edge-vm systemd[1]: skladchik-reports-monitor.service: Failed with result 'exit-code'.\nMONITOR_JOURNAL|Sep 24 19:33:58 edge-vm systemd[1]: Failed to start skladchik-reports-monitor.service - Skladchik XenForo reports monitor.\nMONITOR_JOURNAL|Sep 24 19:34:13 edge-vm systemd[1]: Starting skladchik-reports-monitor.service - Skladchik XenForo reports monitor...\nMONITOR_JOURNAL|Sep 24 19:34:28 edge-vm systemd[1]: skladchik-reports-monitor.service: Main process exited, code=exited, status=1/FAILURE\nMONITOR_JOURNAL|Sep 24 19:34:28 edge-vm systemd[1]: skladchik-reports-monitor.service: Failed with result 'exit-code'.\nMONITOR_JOURNAL|Sep 24 19:34:28 edge-vm systemd[1]: Failed to start skladchik-reports-monitor.service - Skladchik XenForo reports monitor.\nMONITOR_JOURNAL|Sep 24 19:34:44 edge-vm systemd[1]: Starting skladchik-reports-monitor.service - Skladchik XenForo reports monitor...\nMONITOR_JOURNAL|Sep 24 19:34:59 edge-vm systemd[1]: skladchik-reports-monitor.service: Main process exited, code=exited, status=1/FAILURE\nMONITOR_JOURNAL|Sep 24 19:34:59 edge-vm systemd[1]: skladchik-reports-monitor.service: Failed with result 'exit-code'.\nMONITOR_JOURNAL|Sep 24 19:34:59 edge-vm systemd[1]: Failed to start skladchik-reports-monitor.service - Skladchik XenForo reports monitor.\nMONITOR_JOURNAL|Sep 24 19:35:15 edge-vm systemd[1]: Starting skladchik-reports-monitor.service - Skladchik XenForo reports monitor...\nMONITOR_JOURNAL|Sep 24 19:35:30 edge-vm systemd[1]: skladchik-reports-monitor.service: Main process exited, code=exited, status=1/FAILURE\nMONITOR_JOURNAL|Sep 24 19:35:30 edge-vm systemd[1]: skladchik-reports-monitor.service: Failed with result 'exit-code'.\nMONITOR_JOURNAL|Sep 24 19:35:30 edge-vm systemd[1]: Failed to start skladchik-reports-monitor.service - Skladchik XenForo reports monitor.\nMONITOR_JOURNAL|Sep 24 19:35:46 edge-vm systemd[1]: Starting skladchik-reports-monitor.service - Skladchik XenForo reports monitor...\nMONITOR_JOURNAL|Sep 24 19:36:01 edge-vm systemd[1]: skladchik-reports-monitor.service: Main process exited, code=exited, status=1/FAILURE\nMONITOR_JOURNAL|Sep 24 19:36:01 edge-vm systemd[1]: skladchik-reports-monitor.service: Failed with result 'exit-code'.\nMONITOR_JOURNAL|Sep 24 19:36:01 edge-vm systemd[1]: Failed to start skladchik-reports-monitor.service - Skladchik XenForo reports monitor.\nMONITOR_JOURNAL|Sep 24 19:36:17 edge-vm systemd[1]: Starting skladchik-reports-monitor.service - Skladchik XenForo reports monitor...\nMONITOR_JOURNAL|Sep 24 19:36:32 edge-vm systemd[1]: skladchik-reports-monitor.service: Main process exited, code=exited, status=1/FAILURE\nMONITOR_JOURNAL|Sep 24 19:36:32 edge-vm systemd[1]: skladchik-reports-monitor.service: Failed with result 'exit-code'.\nMONITOR_JOURNAL|Sep 24 19:36:32 edge-vm systemd[1]: Failed to start skladchik-reports-monitor.service - Skladchik XenForo reports monitor.\nMONITOR_JOURNAL|Sep 24 19:36:48 edge-vm systemd[1]: Starting skladchik-reports-monitor.service - Skladchik XenForo reports monitor...\nMONITOR_JOURNAL|Sep 24 19:37:03 edge-vm systemd[1]: skladchik-reports-monitor.service: Main process exited, code=exited, status=1/FAILURE\nMONITOR_JOURNAL|Sep 24 19:37:03 edge-vm systemd[1]: skladchik-reports-monitor.service: Failed with result 'exit-code'.\nMONITOR_JOURNAL|Sep 24 19:37:03 edge-vm systemd[1]: Failed to start skladchik-reports-monitor.service - Skladchik XenForo reports monitor.\nMONITOR_JOURNAL|Sep 24 19:37:19 edge-vm systemd[1]: Starting skladchik-reports-monitor.service - Skladchik XenForo reports monitor...\nMONITOR_JOURNAL|Sep 24 19:37:34 edge-vm systemd[1]: skladchik-reports-monitor.service: Main process exited, code=exited, status=1/FAILURE\nMONITOR_JOURNAL|Sep 24 19:37:34 edge-vm systemd[1]: skladchik-reports-monitor.service: Failed with result 'exit-code'.\nMONITOR_JOURNAL|Sep 24 19:37:34 edge-vm systemd[1]: Failed to start skladchik-reports-monitor.service - Skladchik XenForo reports monitor.\nMONITOR_JOURNAL|Sep 24 19:37:50 edge-vm systemd[1]: Starting skladchik-reports-monitor.service - Skladchik XenForo reports monitor...\nMONITOR_JOURNAL|Sep 24 19:38:05 edge-vm systemd[1]: skladchik-reports-monitor.service: Main process exited, code=exited, status=1/FAILURE\nMONITOR_JOURNAL|Sep 24 19:38:05 edge-vm systemd[1]: skladchik-reports-monitor.service: Failed with result 'exit-code'.\nMONITOR_JOURNAL|Sep 24 19:38:05 edge-vm systemd[1]: Failed to start skladchik-reports-monitor.service - Skladchik XenForo reports monitor.\nMONITOR_JOURNAL|Sep 24 19:38:21 edge-vm systemd[1]: Starting skladchik-reports-monitor.service - Skladchik XenForo reports monitor...\nMONITOR_JOURNAL|Sep 24 19:38:36 edge-vm systemd[1]: skladchik-reports-monitor.service: Main process exited, code=exited, status=1/FAILURE\nMONITOR_JOURNAL|Sep 24 19:38:36 edge-vm systemd[1]: skladchik-reports-monitor.service: Failed with result 'exit-code'.\nMONITOR_JOURNAL|Sep 24 19:38:36 edge-vm systemd[1]: Failed to start skladchik-reports-monitor.service - Skladchik XenForo reports monitor.\nMONITOR_JOURNAL|Sep 24 19:38:52 edge-vm systemd[1]: Starting skladchik-reports-monitor.service - Skladchik XenForo reports monitor...\nMONITOR_JOURNAL|Sep 24 19:39:07 edge-vm systemd[1]: skladchik-reports-monitor.service: Main process exited, code=exited, status=1/FAILURE\nMONITOR_JOURNAL|Sep 24 19:39:07 edge-vm systemd[1]: skladchik-reports-monitor.service: Failed with result 'exit-code'.\nMONITOR_JOURNAL|Sep 24 19:39:07 edge-vm systemd[1]: Failed to start skladchik-reports-monitor.service - Skladchik XenForo reports monitor.\nMONITOR_JOURNAL|Sep 24 19:39:22 edge-vm systemd[1]: Starting skladchik-reports-monitor.service - Skladchik XenForo reports monitor...\nMONITOR_JOURNAL|Sep 24 19:39:37 edge-vm systemd[1]: skladchik-reports-monitor.service: Main process exited, code=exited, status=1/FAILURE\nMONITOR_JOURNAL|Sep 24 19:39:37 edge-vm systemd[1]: skladchik-reports-monitor.service: Failed with result 'exit-code'.\nMONITOR_JOURNAL|Sep 24 19:39:37 edge-vm systemd[1]: Failed to start skladchik-reports-monitor.service - Skladchik XenForo reports monitor.\nMONITOR_JOURNAL|Sep 24 19:39:53 edge-vm systemd[1]: Starting skladchik-reports-monitor.service - Skladchik XenForo reports monitor...\nMONITOR_JOURNAL|Sep 24 19:40:08 edge-vm systemd[1]: skladchik-reports-monitor.service: Main process exited, code=exited, status=1/FAILURE\nMONITOR_JOURNAL|Sep 24 19:40:08 edge-vm systemd[1]: skladchik-reports-monitor.service: Failed with result 'exit-code'.\nMONITOR_JOURNAL|Sep 24 19:40:08 edge-vm systemd[1]: Failed to start skladchik-reports-monitor.service - Skladchik XenForo reports monitor.\nMONITOR_JOURNAL|Sep 24 19:40:24 edge-vm systemd[1]: Starting skladchik-reports-monitor.service - Skladchik XenForo reports monitor...\nMONITOR_JOURNAL|Sep 24 19:40:39 edge-vm systemd[1]: skladchik-reports-monitor.service: Main process exited, code=exited, status=1/FAILURE\nMONITOR_JOURNAL|Sep 24 19:40:39 edge-vm systemd[1]: skladchik-reports-monitor.service: Failed with result 'exit-code'.\nMONITOR_JOURNAL|Sep 24 19:40:39 edge-vm systemd[1]: Failed to start skladchik-reports-monitor.service - Skladchik XenForo reports monitor.\nMONITOR_JOURNAL|Sep 24 19:40:55 edge-vm systemd[1]: Starting skladchik-reports-monitor.service - Skladchik XenForo reports monitor...\nMONITOR_JOURNAL|Sep 24 19:41:10 edge-vm systemd[1]: skladchik-reports-monitor.service: Main process exited, code=exited, status=1/FAILURE\nMONITOR_JOURNAL|Sep 24 19:41:10 edge-vm systemd[1]: skladchik-reports-monitor.service: Failed with result 'exit-code'.\nMONITOR_JOURNAL|Sep 24 19:41:10 edge-vm systemd[1]: Failed to start skladchik-reports-monitor.service - Skladchik XenForo reports monitor.\nMONITOR_JOURNAL|Sep 24 19:41:26 edge-vm systemd[1]: Starting skladchik-reports-monitor.service - Skladchik XenForo reports monitor...\nMONITOR_JOURNAL|Sep 24 19:41:41 edge-vm systemd[1]: skladchik-reports-monitor.service: Main process exited, code=exited, status=1/FAILURE\nMONITOR_JOURNAL|Sep 24 19:41:41 edge-vm systemd[1]: skladchik-reports-monitor.service: Failed with result 'exit-code'.\nMONITOR_JOURNAL|Sep 24 19:41:41 edge-vm systemd[1]: Failed to start skladchik-reports-monitor.service - Skladchik XenForo reports monitor.\nMONITOR_JOURNAL|Sep 24 19:41:57 edge-vm systemd[1]: Starting skladchik-reports-monitor.service - Skladchik XenForo reports monitor...\nMONITOR_JOURNAL|Sep 24 19:42:12 edge-vm systemd[1]: skladchik-reports-monitor.service: Main process exited, code=exited, status=1/FAILURE\nMONITOR_JOURNAL|Sep 24 19:42:12 edge-vm systemd[1]: skladchik-reports-monitor.service: Failed with result 'exit-code'.\nMONITOR_JOURNAL|Sep 24 19:42:12 edge-vm systemd[1]: Failed to start skladchik-reports-monitor.service - Skladchik XenForo reports monitor.\nMONITOR_JOURNAL|Sep 24 19:42:28 edge-vm systemd[1]: Starting skladchik-reports-monitor.service - Skladchik XenForo reports monitor...\nMONITOR_JOURNAL|Sep 24 19:42:43 edge-vm systemd[1]: skladchik-reports-monitor.service: Main process exited, code=exited, status=1/FAILURE\nMONITOR_JOURNAL|Sep 24 19:42:43 edge-vm systemd[1]: skladchik-reports-monitor.service: Failed with result 'exit-code'.\nMONITOR_JOURNAL|Sep 24 19:42:43 edge-vm systemd[1]: Failed to start skladchik-reports-monitor.service - Skladchik XenForo reports monitor.\nMONITOR_JOURNAL|Sep 24 19:42:58 edge-vm systemd[1]: Starting skladchik-reports-monitor.service - Skladchik XenForo reports monitor...\nMONITOR_JOURNAL|Sep 24 19:43:13 edge-vm systemd[1]: skladchik-reports-monitor.service: Main process exited, code=exited, status=1/FAILURE\nMONITOR_JOURNAL|Sep 24 19:43:13 edge-vm systemd[1]: skladchik-reports-monitor.service: Failed with result 'exit-code'.\nMONITOR_JOURNAL|Sep 24 19:43:13 edge-vm systemd[1]: Failed to start skladchik-reports-monitor.service - Skladchik XenForo reports monitor.\nMONITOR_JOURNAL|Sep 24 19:43:29 edge-vm systemd[1]: Starting skladchik-reports-monitor.service - Skladchik XenForo reports monitor...\nMONITOR_JOURNAL|Sep 24 19:43:44 edge-vm systemd[1]: skladchik-reports-monitor.service: Main process exited, code=exited, status=1/FAILURE\nMONITOR_JOURNAL|Sep 24 19:43:44 edge-vm systemd[1]: skladchik-reports-monitor.service: Failed with result 'exit-code'.\nMONITOR_JOURNAL|Sep 24 19:43:44 edge-vm systemd[1]: Failed to start skladchik-reports-monitor.service - Skladchik XenForo reports monitor.\nMONITOR_JOURNAL|Sep 24 19:44:00 edge-vm systemd[1]: Starting skladchik-reports-monitor.service - Skladchik XenForo reports monitor...\nMONITOR_JOURNAL|Sep 24 19:44:15 edge-vm systemd[1]: skladchik-reports-monitor.service: Main process exited, code=exited, status=1/FAILURE\nMONITOR_JOURNAL|Sep 24 19:44:15 edge-vm systemd[1]: skladchik-reports-monitor.service: Failed with result 'exit-code'.\nMONITOR_JOURNAL|Sep 24 19:44:15 edge-vm systemd[1]: Failed to start skladchik-reports-monitor.service - Skladchik XenForo reports monitor.\nMONITOR_JOURNAL|Sep 24 19:44:31 edge-vm systemd[1]: Starting skladchik-reports-monitor.service - Skladchik XenForo reports monitor...\nMONITOR_JOURNAL|Sep 24 19:44:46 edge-vm systemd[1]: skladchik-reports-monitor.service: Main process exited, code=exited, status=1/FAILURE\nMONITOR_JOURNAL|Sep 24 19:44:46 edge-vm systemd[1]: skladchik-reports-monitor.service: Failed with result 'exit-code'.\nMONITOR_JOURNAL|Sep 24 19:44:46 edge-vm systemd[1]: Failed to start skladchik-reports-monitor.service - Skladchik XenForo reports monitor.\n=== WATCHDOG_RECENT_JOURNAL_FILTERED ===\nWATCHDOG_JOURNAL|Sep 24 18:32:04 edge-vm python3[449857]: WATCHDOG_STATUS=ALERT\nWATCHDOG_JOURNAL|Sep 24 18:32:04 edge-vm python3[449857]: PROBLEMS=monitor_status_not_ok:ERROR;monitor_reports_bad:missing;netbird_route_not_wt0\nWATCHDOG_JOURNAL|Sep 24 18:37:04 edge-vm python3[453332]: WATCHDOG_STATUS=ALERT\nWATCHDOG_JOURNAL|Sep 24 18:37:04 edge-vm python3[453332]: PROBLEMS=monitor_status_not_ok:ERROR;monitor_reports_bad:missing;netbird_route_not_wt0\nWATCHDOG_JOURNAL|Sep 24 18:42:04 edge-vm python3[456699]: WATCHDOG_STATUS=ALERT\nWATCHDOG_JOURNAL|Sep 24 18:42:04 edge-vm python3[456699]: PROBLEMS=monitor_status_not_ok:ERROR;monitor_reports_bad:missing;netbird_route_not_wt0\nWATCHDOG_JOURNAL|Sep 24 18:47:04 edge-vm python3[460010]: WATCHDOG_STATUS=ALERT\nWATCHDOG_JOURNAL|Sep 24 18:47:04 edge-vm python3[460010]: PROBLEMS=monitor_status_not_ok:ERROR;monitor_reports_bad:missing;netbird_route_not_wt0\nWATCHDOG_JOURNAL|Sep 24 18:52:06 edge-vm python3[463379]: WATCHDOG_STATUS=ALERT\nWATCHDOG_JOURNAL|Sep 24 18:52:06 edge-vm python3[463379]: PROBLEMS=monitor_status_not_ok:ERROR;monitor_reports_bad:missing;netbird_route_not_wt0\nWATCHDOG_JOURNAL|Sep 24 18:57:12 edge-vm python3[466658]: WATCHDOG_STATUS=ALERT\nWATCHDOG_JOURNAL|Sep 24 18:57:12 edge-vm python3[466658]: PROBLEMS=monitor_status_not_ok:ERROR;monitor_reports_bad:missing;netbird_route_not_wt0\nWATCHDOG_JOURNAL|Sep 24 19:02:19 edge-vm python3[470018]: WATCHDOG_STATUS=ALERT\nWATCHDOG_JOURNAL|Sep 24 19:02:19 edge-vm python3[470018]: PROBLEMS=monitor_status_not_ok:ERROR;monitor_reports_bad:missing;netbird_route_not_wt0\nWATCHDOG_JOURNAL|Sep 24 19:07:21 edge-vm python3[473503]: WATCHDOG_STATUS=ALERT\nWATCHDOG_JOURNAL|Sep 24 19:07:21 edge-vm python3[473503]: PROBLEMS=monitor_status_not_ok:ERROR;monitor_reports_bad:missing;netbird_route_not_wt0\nWATCHDOG_JOURNAL|Sep 24 19:12:21 edge-vm python3[476873]: WATCHDOG_STATUS=ALERT\nWATCHDOG_JOURNAL|Sep 24 19:12:21 edge-vm python3[476873]: PROBLEMS=monitor_status_not_ok:ERROR;monitor_reports_bad:missing;netbird_route_not_wt0\nWATCHDOG_JOURNAL|Sep 24 19:17:21 edge-vm python3[477705]: WATCHDOG_STATUS=ALERT\nWATCHDOG_JOURNAL|Sep 24 19:17:21 edge-vm python3[477705]: PROBLEMS=monitor_status_not_ok:ERROR;monitor_reports_bad:missing;netbird_route_not_wt0\nWATCHDOG_JOURNAL|Sep 24 19:22:21 edge-vm python3[481219]: WATCHDOG_STATUS=ALERT\nWATCHDOG_JOURNAL|Sep 24 19:22:21 edge-vm python3[481219]: PROBLEMS=monitor_status_not_ok:ERROR;monitor_reports_bad:missing;netbird_route_not_wt0\nWATCHDOG_JOURNAL|Sep 24 19:27:21 edge-vm python3[484527]: WATCHDOG_STATUS=ALERT\nWATCHDOG_JOURNAL|Sep 24 19:27:21 edge-vm python3[484527]: PROBLEMS=monitor_status_not_ok:ERROR;monitor_reports_bad:missing;netbird_route_not_wt0\nWATCHDOG_JOURNAL|Sep 24 19:32:21 edge-vm python3[487881]: WATCHDOG_STATUS=ALERT\nWATCHDOG_JOURNAL|Sep 24 19:32:21 edge-vm python3[487881]: PROBLEMS=monitor_status_not_ok:ERROR;monitor_reports_bad:missing;netbird_route_not_wt0\nWATCHDOG_JOURNAL|Sep 24 19:37:22 edge-vm python3[491520]: WATCHDOG_STATUS=ALERT\nWATCHDOG_JOURNAL|Sep 24 19:37:22 edge-vm python3[491520]: PROBLEMS=monitor_status_not_ok:ERROR;monitor_reports_bad:missing;netbird_route_not_wt0\nWATCHDOG_JOURNAL|Sep 24 19:42:22 edge-vm python3[495021]: WATCHDOG_STATUS=ALERT\nWATCHDOG_JOURNAL|Sep 24 19:42:22 edge-vm python3[495021]: PROBLEMS=monitor_status_not_ok:ERROR;monitor_reports_bad:missing;netbird_route_not_wt0\n=== STATE_DIR_METADATA ===\nSTATE_FILE|.session-keepalive-installed\t0\t1783667766.2769201920\nSTATE_FILE|age-alert-health.txt\t73\t1790232322.4277234210\nSTATE_FILE|alert-health.txt\t530\t1790268142.7951166440\nSTATE_FILE|alert-state.json\t305\t1790268142.7951166440\nSTATE_FILE|cookies.jar\t744\t1790268286.5632246860\nSTATE_FILE|event-selftest-health.txt\t168\t1788243905.4037463360\nSTATE_FILE|health.txt\t506\t1790268286.5632246860\nSTATE_FILE|state.sqlite\t577536\t1790232322.4277234210\nSTATE_FILE|weekly-summary-health.txt\t118\t1789972505.6221476670\nEDGE_MONITOR_RCA_END=1\nEDGE_QGA_GUEST_STDERR_PRESENT=false\nRCA_HEALTH_STATUS=ERROR\nRCA_REPORTS=MISSING\nRCA_SESSION_KEEPALIVE=persistent_cookie_jar\nRCA_COOKIE_JAR_STATE=yes\nRCA_COOKIE_EXPIRY_STATUS=OK\nRCA_AUTH_JOURNAL_HITS=0\nRCA_NETWORK_JOURNAL_HITS=0\nRCA_RUNTIME_ERROR_HITS=60\nRCA_HTTP_BLOCK_HITS=0\nDECISION=PROVEN_OR_STRONGLY_INDICATED_SKLADCHIK_MONITOR_RUNTIME_FAILURE\nNEXT_GATE=TARGETED_MONITOR_RUNTIME_RCA_READ_ONLY\nTASK_RESULT=PASS_SKLADCHIK_MONITOR_RCA_RUNTIME\nHOMELAB_RESULT_CONTRACT={\"version\":1,\"command_id\":\"SUPPORT-260924-SKLADCHIK-MONITOR-STATUS-ERROR-RCA-73167R1\",\"status\":\"OK\",\"changes_made\":false,\"rollback_started\":false,\"rollback_restored\":null}\nWORKERS2_SKLADCHIK_MONITOR_STATUS_ERROR_RCA_END=1\n" + "output": "DNSD01_9166_BEGIN=true\nRCA_FIX=EXPAND_CLOUDFLARE_AUTH_DISCOVERY_ACROSS_PVE_NODES_RUNNING_LXC_QEMU_AND_EDGE\nMUTATION_BOUNDARY=READ_ONLY_CLOUDFLARE_AND_PUBLIC_DNS_INVENTORY;NO_DNS_RECORD_NS_DS_OR_SITE_CHANGE\nPREVIOUS_EXACT_9165=PASS\nFRESH_ASSISTANT_RUNTIME_HEAD=726e443a1d83c2009482e32ca0301f603ea60f7f\nD01_SOURCE_PVE01_RC=0\nD01_SOURCE_PVE01_LINE_COUNT=6\nD01_SOURCE_PVE02_RC=0\nD01_SOURCE_PVE02_LINE_COUNT=1\nD01_SOURCE_PVE03_RC=0\nD01_SOURCE_PVE03_LINE_COUNT=5\nD01_SOURCE_EDGE_RC=0\nD01_SOURCE_EDGE_LINE_COUNT=3\nD01_CLUSTER_AUTH_RAW_LINE_COUNT=15\nD01_CF_AUTH_DISCOVERY=PASS\nD01_CF_AUTH_CANDIDATE_COUNT=4\nD01_AUTH_1_KIND=BEARER\n[REDACTED]\nD01_AUTH_1_DNS_RECORD_READ_ZONE_COUNT=0\nD01_AUTH_1_DNSSEC_READ_ZONE_COUNT=0\nD01_AUTH_2_KIND=BEARER\n[REDACTED]\nD01_AUTH_2_DNS_RECORD_READ_ZONE_COUNT=1\nD01_AUTH_2_DNSSEC_READ_ZONE_COUNT=1\nD01_AUTH_3_KIND=BEARER\n[REDACTED]\nD01_AUTH_3_DNS_RECORD_READ_ZONE_COUNT=1\nD01_AUTH_3_DNSSEC_READ_ZONE_COUNT=1\nD01_AUTH_4_KIND=BEARER\n[REDACTED]\nD01_AUTH_4_DNS_RECORD_READ_ZONE_COUNT=5\nD01_AUTH_4_DNSSEC_READ_ZONE_COUNT=5\nD01_CF_ACCESSIBLE_ZONE_COUNT=13\nD01_EXPECTED_ZONE_COUNT=13\nD01_EXPECTED_MISSING_COUNT=0\nD01_EXPECTED_MISSING=NONE\nD01_EXTRA_ZONE_COUNT=0\nD01_EXTRA_ZONES=NONE\nD01_ZONESET_SHA256=766204ad5d819bfa7cf96cc42b8fbde659529b923b9b4aaa32c1c83d1bd9649e\nD01_DNS_RECORD_READABLE_ZONE_COUNT=7\nD01_DNS_RECORD_UNREADABLE_ZONE_COUNT=6\nD01_DNS_RECORD_UNREADABLE_ZONES=hapusya.ru,hbarhub.ru,kingofwolk.ru,newfi.ru,newfilya.ru,pvepro.ru\nD01_EXPECTED_DNS_UNREADABLE_COUNT=6\nD01_EXPECTED_DNS_UNREADABLE=hapusya.ru,hbarhub.ru,kingofwolk.ru,newfi.ru,newfilya.ru,pvepro.ru\nD01_COMPLETENESS=INCOMPLETE_CLOUDFLARE_SCOPE_OR_DNS_READ\nTASK_COMPLETE=true\nTASK_RESULT=FAIL_DNS_D01_CLUSTER_COVERAGE\nDECISION=D01_DNS_RECORD_READ_SCOPE_STILL_INCOMPLETE\nNEXT_GATE=READ_EXACT_9166_AND_FIX_ONLY_D01_PROVEN_BLOCKER\nHOMELAB_RESULT_CONTRACT={\"version\":1,\"command_id\":\"SUPPORT-260924-BEGET-DNS-D01-CLUSTER-COVERAGE-9166R1\",\"status\":\"FAIL\",\"changes_made\":false,\"rollback_started\":false,\"rollback_restored\":null}\n" } diff --git a/runtime/latest.txt b/runtime/latest.txt index a15fd5d5..04c8f9e9 100644 --- a/runtime/latest.txt +++ b/runtime/latest.txt @@ -1,22 +1,22 @@ CHAT_OUTPUT_BEGIN -COMMAND_ID=SUPPORT-260924-SKLADCHIK-MONITOR-STATUS-ERROR-RCA-73167R1 -STATUS=OK -RC=0 +COMMAND_ID=SUPPORT-260924-BEGET-DNS-D01-CLUSTER-COVERAGE-9166R1 +STATUS=FAIL +RC=42 HOST=pve01 MODE=read-only -COMPONENT=skladchik-monitor-status-error-rca +COMPONENT=beget-dns-d01-cloudflare-inventory REFERENCE_REGISTER_CHECK=OK REFERENCE_SHA256=5a3d8e5154c41cb582a4e0aca68090be1f0138918bf82131a948df326f9d8d66 ERROR_REGISTER_CHECK=OK ERROR_REGISTER_SHA256=3b09a553ec0f527ed3afeed4753f52a74ee3036045bbfb3c685e9f8af4ba7ba0 -COMMAND_SHA256=6f2291c89dca3072b8ba3b29c129d594d18708afc9bb806b08dad46f6410d42f +COMMAND_SHA256=3fdfe7b2879eee2cf217828c2f81f14d30158e549f0fdddb53c1f9dbe1ff074f DUPLICATE_FAILED_COMMAND_BLOCKED=false EXECUTION_STARTED=true CHANGE_DECLARED=false RESULT_CONTRACT_VALID=true RESULT_CONTRACT_STATUS=NOT_APPLICABLE RESULT_CONTRACT_ERROR=NONE -COMMAND_RC=0 +COMMAND_RC=42 CHANGES_MADE=false ROLLBACK_STARTED=false ROLLBACK_RESTORED=null @@ -24,291 +24,59 @@ MUTATION_OUTCOME=NO_MUTATION SANITIZED=yes SECRETS_INCLUDED=no PRIVATE_ADDRESSES_INCLUDED=no -RAW_EVIDENCE_SHA256=0a9a4ccd3752109628076c7847ace825bed2ce42c2bff682fec4e9e16fd70016 -SANITIZED_OUTPUT_SHA256=0a9a4ccd3752109628076c7847ace825bed2ce42c2bff682fec4e9e16fd70016 +RAW_EVIDENCE_SHA256=5a92c486b9fa5324f80d8a7592d804b03f89ac043571bf303eb0b376567273c6 +SANITIZED_OUTPUT_SHA256=602d96b59e5b34e3ca818c05c9e047d90ecca7db5a081311cabafddd99f0a241 OUTPUT_BEGIN -WORKERS2_SKLADCHIK_MONITOR_STATUS_ERROR_RCA_BEGIN=1 -COMMAND_ID=SUPPORT-260924-SKLADCHIK-MONITOR-STATUS-ERROR-RCA-73167R1 -MODE=read-only -COMPONENT=skladchik-monitor-status-error-rca -MUTATION_BOUNDARY=NONE;READ_ONLY_EDGE_QGA_SERVICE_HEALTH_COOKIE_METADATA_SOURCE_HASH_AND_FILTERED_JOURNAL;TEMP_/run_ONLY;NO_MONITOR_START;NO_SERVICE_ACTION;NO_COOKIE_CONTENT;NO_CONFIG_SECRET_VALUES;NO_NETBIRD_CHANGE;NO_ROUTE_CHANGE;NO_GIT_WRITE -PREDECESSOR_EXACT=SUPPORT-260924-SKLADCHIK-SUPERVISOR-ROUTE-INVARIANT-REPAIR-73166R1 -PREDECESSOR_DECISION=PASS_ROUTE_INVARIANT_REPAIRED_MONITOR_ERROR_REMAINS -REFERENCE_SHA256=5a3d8e5154c41cb582a4e0aca68090be1f0138918bf82131a948df326f9d8d66 -ERROR_REGISTER_SHA256=3b09a553ec0f527ed3afeed4753f52a74ee3036045bbfb3c685e9f8af4ba7ba0 -RUNNER_SHA256=b248a4c32c9cc64e5747e7dce6c7fc0a23f5124a77c71ce72e27a81aceae9d2d -EDGE_QGA_TRANSPORT_RC=0 -EDGE_QGA_TRANSPORT_STDERR_SHA256=e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855 -EDGE_QGA_JSON_PARSE=true -EDGE_QGA_EXITCODE=0 -EDGE_MONITOR_RCA_BEGIN=1 -EDGE_HOSTNAME=edge-vm -EDGE_TIME_UTC=2026-09-24T16:44:59Z -skladchik_reports_monitor_service_ACTIVE=failed -skladchik_reports_monitor_service_ENABLED=static -skladchik_reports_monitor_service_RESULT=exit-code -skladchik_reports_monitor_service_EXEC_MAIN_STATUS=1 -skladchik_reports_monitor_timer_ACTIVE=active -skladchik_reports_monitor_timer_ENABLED=enabled -skladchik_reports_monitor_timer_RESULT=success -skladchik_reports_monitor_timer_EXEC_MAIN_STATUS= -skladchik_reports_monitor_watchdog_timer_ACTIVE=active -skladchik_reports_monitor_watchdog_timer_ENABLED=enabled -skladchik_reports_monitor_watchdog_timer_RESULT=success -skladchik_reports_monitor_watchdog_timer_EXEC_MAIN_STATUS= -skladchik_reports_monitor_heartbeat_timer_ACTIVE=active -skladchik_reports_monitor_heartbeat_timer_ENABLED=enabled -skladchik_reports_monitor_heartbeat_timer_RESULT=success -skladchik_reports_monitor_heartbeat_timer_EXEC_MAIN_STATUS= -EDGE_HEALTH_EXISTS=true -EDGE_HEALTH_SIZE=506 -EDGE_HEALTH_MODE=640 -EDGE_HEALTH_OWNER=0:0 -EDGE_HEALTH_MTIME_EPOCH=1790268286 -EDGE_HEALTH_SHA256=6db70fe9833f5d47a07873cc8c8a2a8ee07e4456b74985efc4c0d69d373ba2f2 -EDGE_ALERT_HEALTH_EXISTS=true -EDGE_ALERT_HEALTH_SIZE=530 -EDGE_ALERT_HEALTH_MODE=644 -EDGE_ALERT_HEALTH_OWNER=0:0 -EDGE_ALERT_HEALTH_MTIME_EPOCH=1790268142 -EDGE_ALERT_HEALTH_SHA256=d1493c44f34d1aecfae8e35dbee4e4cace3236088921b1a69c786f4c793cc06e -EDGE_COOKIE_JAR_EXISTS=true -EDGE_COOKIE_JAR_SIZE=744 -EDGE_COOKIE_JAR_MODE=600 -EDGE_COOKIE_JAR_OWNER=0:0 -EDGE_COOKIE_JAR_MTIME_EPOCH=1790268286 -EDGE_COOKIE_JAR_SHA256=c24634109cab376eab1afe3b3de727d2a7f5b16702ed98f6ee516a3935973e7c -EDGE_CONFIG_EXISTS=true -EDGE_CONFIG_SIZE=292 -EDGE_CONFIG_MODE=640 -EDGE_CONFIG_OWNER=0:0 -EDGE_CONFIG_MTIME_EPOCH=1784363565 -EDGE_CONFIG_SHA256=911b8d72a9ca8eb2f4aba14111d5a8acc88898aa0b89de36d7557f0ca10787fd -EDGE_SECRETS_ENV_EXISTS=true -EDGE_SECRETS_ENV_SIZE=513 -EDGE_SECRETS_ENV_MODE=600 -EDGE_SECRETS_ENV_OWNER=0:0 -EDGE_SECRETS_ENV_MTIME_EPOCH=1790189249 -EDGE_SECRETS_ENV_SHA256=f2e3757eb9e6f4fd352cf38e4d99ad080d824df5b6916df64dfd036e662ee23b -EDGE_MONITOR_PY_EXISTS=true -EDGE_MONITOR_PY_SIZE=25677 -EDGE_MONITOR_PY_MODE=700 -EDGE_MONITOR_PY_OWNER=0:0 -EDGE_MONITOR_PY_MTIME_EPOCH=1786729896 -EDGE_MONITOR_PY_SHA256=bf3a47cf3c53a4efa1f49322f96203a24b9077ea64547162e64c5d3d3326d0ff -HEALTH_SAFE_BEGIN=1 -HEALTH_KV|STATUS=ERROR -HEALTH_KV|TS=2026-09-24T16:44:46Z -HEALTH_KV|error=MonitorError('curl_failed rc=28 stderr=curl: (28) Connection timed out after 15003 milliseconds\n') -HEALTH_KV|session_keepalive=persistent_cookie_jar -HEALTH_KV|cookie_jar=yes -HEALTH_KV|cookie_jar_bytes=744 -HEALTH_KV|cookie_persistent_count=1 -HEALTH_KV|cookie_expiry_utc=2026-10-23T18:47:24Z -HEALTH_KV|cookie_expires_in_seconds=2512958 -HEALTH_KV|cookie_expires_in_days=29 -HEALTH_KV|cookie_expiry_status=OK -HEALTH_KEYS=STATUS,TS,cookie_earliest_expiry_utc,cookie_expires_in_days,cookie_expires_in_seconds,cookie_expiry_status,cookie_expiry_utc,cookie_expiry_warning_days,cookie_jar,cookie_jar_bytes,cookie_jar_mtime,cookie_persistent_count,cookie_session_count,error,session_keepalive -HEALTH_SAFE_END=1 -ALERT_SAFE_BEGIN=1 -ALERT_KV|STATUS=ALERT -ALERT_KV|TS=2026-09-24T16:42:22Z -ALERT_KV|monitor_status=ERROR -ALERT_KV|session_keepalive=persistent_cookie_jar -ALERT_KV|cookie_jar=yes -ALERT_KV|cookie_jar_bytes=744 -ALERT_KV|cookie_expiry_status=OK -ALERT_KV|cookie_expiry_utc=2026-10-23T18:47:24Z -ALERT_KV|cookie_expires_in_seconds=2513112 -ALERT_KV|cookie_expires_in_days=29 -ALERT_KEYS=STATUS,TS,cookie_expires_in_days,cookie_expires_in_seconds,cookie_expiry_status,cookie_expiry_utc,cookie_jar,cookie_jar_bytes,health_age_seconds,mail_sent,monitor_open,monitor_reports,monitor_resolved,monitor_status,monitor_timer_active,monitor_timer_enabled,netbird_route_ok,problems,session_keepalive -ALERT_SAFE_END=1 -MONITOR_AST_PARSE_OK=true -MONITOR_SOURCE_AUTH_MATCH_COUNT=23 -MONITOR_SOURCE_COOKIE_MATCH_COUNT=63 -MONITOR_SOURCE_SESSION_MATCH_COUNT=7 -MONITOR_SOURCE_CAPTCHA_MATCH_COUNT=0 -MONITOR_SOURCE_HTTP_MATCH_COUNT=10 -MONITOR_SOURCE_STATUS_ERROR_MATCH_COUNT=1 -MONITOR_SOURCE_HEALTH_WRITE_MATCH_COUNT=1 -=== MONITOR_RECENT_JOURNAL_FILTERED === -MONITOR_JOURNAL|Sep 24 19:29:36 edge-vm systemd[1]: Starting skladchik-reports-monitor.service - Skladchik XenForo reports monitor... -MONITOR_JOURNAL|Sep 24 19:29:51 edge-vm systemd[1]: skladchik-reports-monitor.service: Main process exited, code=exited, status=1/FAILURE -MONITOR_JOURNAL|Sep 24 19:29:51 edge-vm systemd[1]: skladchik-reports-monitor.service: Failed with result 'exit-code'. -MONITOR_JOURNAL|Sep 24 19:29:51 edge-vm systemd[1]: Failed to start skladchik-reports-monitor.service - Skladchik XenForo reports monitor. -MONITOR_JOURNAL|Sep 24 19:30:07 edge-vm systemd[1]: Starting skladchik-reports-monitor.service - Skladchik XenForo reports monitor... -MONITOR_JOURNAL|Sep 24 19:30:22 edge-vm systemd[1]: skladchik-reports-monitor.service: Main process exited, code=exited, status=1/FAILURE -MONITOR_JOURNAL|Sep 24 19:30:22 edge-vm systemd[1]: skladchik-reports-monitor.service: Failed with result 'exit-code'. -MONITOR_JOURNAL|Sep 24 19:30:22 edge-vm systemd[1]: Failed to start skladchik-reports-monitor.service - Skladchik XenForo reports monitor. -MONITOR_JOURNAL|Sep 24 19:30:38 edge-vm systemd[1]: Starting skladchik-reports-monitor.service - Skladchik XenForo reports monitor... -MONITOR_JOURNAL|Sep 24 19:30:53 edge-vm systemd[1]: skladchik-reports-monitor.service: Main process exited, code=exited, status=1/FAILURE -MONITOR_JOURNAL|Sep 24 19:30:53 edge-vm systemd[1]: skladchik-reports-monitor.service: Failed with result 'exit-code'. -MONITOR_JOURNAL|Sep 24 19:30:53 edge-vm systemd[1]: Failed to start skladchik-reports-monitor.service - Skladchik XenForo reports monitor. -MONITOR_JOURNAL|Sep 24 19:31:09 edge-vm systemd[1]: Starting skladchik-reports-monitor.service - Skladchik XenForo reports monitor... -MONITOR_JOURNAL|Sep 24 19:31:24 edge-vm systemd[1]: skladchik-reports-monitor.service: Main process exited, code=exited, status=1/FAILURE -MONITOR_JOURNAL|Sep 24 19:31:24 edge-vm systemd[1]: skladchik-reports-monitor.service: Failed with result 'exit-code'. -MONITOR_JOURNAL|Sep 24 19:31:24 edge-vm systemd[1]: Failed to start skladchik-reports-monitor.service - Skladchik XenForo reports monitor. -MONITOR_JOURNAL|Sep 24 19:31:40 edge-vm systemd[1]: Starting skladchik-reports-monitor.service - Skladchik XenForo reports monitor... -MONITOR_JOURNAL|Sep 24 19:31:55 edge-vm systemd[1]: skladchik-reports-monitor.service: Main process exited, code=exited, status=1/FAILURE -MONITOR_JOURNAL|Sep 24 19:31:55 edge-vm systemd[1]: skladchik-reports-monitor.service: Failed with result 'exit-code'. -MONITOR_JOURNAL|Sep 24 19:31:55 edge-vm systemd[1]: Failed to start skladchik-reports-monitor.service - Skladchik XenForo reports monitor. -MONITOR_JOURNAL|Sep 24 19:32:11 edge-vm systemd[1]: Starting skladchik-reports-monitor.service - Skladchik XenForo reports monitor... -MONITOR_JOURNAL|Sep 24 19:32:26 edge-vm systemd[1]: skladchik-reports-monitor.service: Main process exited, code=exited, status=1/FAILURE -MONITOR_JOURNAL|Sep 24 19:32:26 edge-vm systemd[1]: skladchik-reports-monitor.service: Failed with result 'exit-code'. -MONITOR_JOURNAL|Sep 24 19:32:26 edge-vm systemd[1]: Failed to start skladchik-reports-monitor.service - Skladchik XenForo reports monitor. -MONITOR_JOURNAL|Sep 24 19:32:42 edge-vm systemd[1]: Starting skladchik-reports-monitor.service - Skladchik XenForo reports monitor... -MONITOR_JOURNAL|Sep 24 19:32:57 edge-vm systemd[1]: skladchik-reports-monitor.service: Main process exited, code=exited, status=1/FAILURE -MONITOR_JOURNAL|Sep 24 19:32:57 edge-vm systemd[1]: skladchik-reports-monitor.service: Failed with result 'exit-code'. -MONITOR_JOURNAL|Sep 24 19:32:57 edge-vm systemd[1]: Failed to start skladchik-reports-monitor.service - Skladchik XenForo reports monitor. -MONITOR_JOURNAL|Sep 24 19:33:12 edge-vm systemd[1]: Starting skladchik-reports-monitor.service - Skladchik XenForo reports monitor... -MONITOR_JOURNAL|Sep 24 19:33:27 edge-vm systemd[1]: skladchik-reports-monitor.service: Main process exited, code=exited, status=1/FAILURE -MONITOR_JOURNAL|Sep 24 19:33:27 edge-vm systemd[1]: skladchik-reports-monitor.service: Failed with result 'exit-code'. -MONITOR_JOURNAL|Sep 24 19:33:27 edge-vm systemd[1]: Failed to start skladchik-reports-monitor.service - Skladchik XenForo reports monitor. -MONITOR_JOURNAL|Sep 24 19:33:43 edge-vm systemd[1]: Starting skladchik-reports-monitor.service - Skladchik XenForo reports monitor... -MONITOR_JOURNAL|Sep 24 19:33:58 edge-vm systemd[1]: skladchik-reports-monitor.service: Main process exited, code=exited, status=1/FAILURE -MONITOR_JOURNAL|Sep 24 19:33:58 edge-vm systemd[1]: skladchik-reports-monitor.service: Failed with result 'exit-code'. -MONITOR_JOURNAL|Sep 24 19:33:58 edge-vm systemd[1]: Failed to start skladchik-reports-monitor.service - Skladchik XenForo reports monitor. -MONITOR_JOURNAL|Sep 24 19:34:13 edge-vm systemd[1]: Starting skladchik-reports-monitor.service - Skladchik XenForo reports monitor... -MONITOR_JOURNAL|Sep 24 19:34:28 edge-vm systemd[1]: skladchik-reports-monitor.service: Main process exited, code=exited, status=1/FAILURE -MONITOR_JOURNAL|Sep 24 19:34:28 edge-vm systemd[1]: skladchik-reports-monitor.service: Failed with result 'exit-code'. -MONITOR_JOURNAL|Sep 24 19:34:28 edge-vm systemd[1]: Failed to start skladchik-reports-monitor.service - Skladchik XenForo reports monitor. -MONITOR_JOURNAL|Sep 24 19:34:44 edge-vm systemd[1]: Starting skladchik-reports-monitor.service - Skladchik XenForo reports monitor... -MONITOR_JOURNAL|Sep 24 19:34:59 edge-vm systemd[1]: skladchik-reports-monitor.service: Main process exited, code=exited, status=1/FAILURE -MONITOR_JOURNAL|Sep 24 19:34:59 edge-vm systemd[1]: skladchik-reports-monitor.service: Failed with result 'exit-code'. -MONITOR_JOURNAL|Sep 24 19:34:59 edge-vm systemd[1]: Failed to start skladchik-reports-monitor.service - Skladchik XenForo reports monitor. -MONITOR_JOURNAL|Sep 24 19:35:15 edge-vm systemd[1]: Starting skladchik-reports-monitor.service - Skladchik XenForo reports monitor... -MONITOR_JOURNAL|Sep 24 19:35:30 edge-vm systemd[1]: skladchik-reports-monitor.service: Main process exited, code=exited, status=1/FAILURE -MONITOR_JOURNAL|Sep 24 19:35:30 edge-vm systemd[1]: skladchik-reports-monitor.service: Failed with result 'exit-code'. -MONITOR_JOURNAL|Sep 24 19:35:30 edge-vm systemd[1]: Failed to start skladchik-reports-monitor.service - Skladchik XenForo reports monitor. -MONITOR_JOURNAL|Sep 24 19:35:46 edge-vm systemd[1]: Starting skladchik-reports-monitor.service - Skladchik XenForo reports monitor... -MONITOR_JOURNAL|Sep 24 19:36:01 edge-vm systemd[1]: skladchik-reports-monitor.service: Main process exited, code=exited, status=1/FAILURE -MONITOR_JOURNAL|Sep 24 19:36:01 edge-vm systemd[1]: skladchik-reports-monitor.service: Failed with result 'exit-code'. -MONITOR_JOURNAL|Sep 24 19:36:01 edge-vm systemd[1]: Failed to start skladchik-reports-monitor.service - Skladchik XenForo reports monitor. -MONITOR_JOURNAL|Sep 24 19:36:17 edge-vm systemd[1]: Starting skladchik-reports-monitor.service - Skladchik XenForo reports monitor... -MONITOR_JOURNAL|Sep 24 19:36:32 edge-vm systemd[1]: skladchik-reports-monitor.service: Main process exited, code=exited, status=1/FAILURE -MONITOR_JOURNAL|Sep 24 19:36:32 edge-vm systemd[1]: skladchik-reports-monitor.service: Failed with result 'exit-code'. -MONITOR_JOURNAL|Sep 24 19:36:32 edge-vm systemd[1]: Failed to start skladchik-reports-monitor.service - Skladchik XenForo reports monitor. -MONITOR_JOURNAL|Sep 24 19:36:48 edge-vm systemd[1]: Starting skladchik-reports-monitor.service - Skladchik XenForo reports monitor... -MONITOR_JOURNAL|Sep 24 19:37:03 edge-vm systemd[1]: skladchik-reports-monitor.service: Main process exited, code=exited, status=1/FAILURE -MONITOR_JOURNAL|Sep 24 19:37:03 edge-vm systemd[1]: skladchik-reports-monitor.service: Failed with result 'exit-code'. -MONITOR_JOURNAL|Sep 24 19:37:03 edge-vm systemd[1]: Failed to start skladchik-reports-monitor.service - Skladchik XenForo reports monitor. -MONITOR_JOURNAL|Sep 24 19:37:19 edge-vm systemd[1]: Starting skladchik-reports-monitor.service - Skladchik XenForo reports monitor... -MONITOR_JOURNAL|Sep 24 19:37:34 edge-vm systemd[1]: skladchik-reports-monitor.service: Main process exited, code=exited, status=1/FAILURE -MONITOR_JOURNAL|Sep 24 19:37:34 edge-vm systemd[1]: skladchik-reports-monitor.service: Failed with result 'exit-code'. -MONITOR_JOURNAL|Sep 24 19:37:34 edge-vm systemd[1]: Failed to start skladchik-reports-monitor.service - Skladchik XenForo reports monitor. -MONITOR_JOURNAL|Sep 24 19:37:50 edge-vm systemd[1]: Starting skladchik-reports-monitor.service - Skladchik XenForo reports monitor... -MONITOR_JOURNAL|Sep 24 19:38:05 edge-vm systemd[1]: skladchik-reports-monitor.service: Main process exited, code=exited, status=1/FAILURE -MONITOR_JOURNAL|Sep 24 19:38:05 edge-vm systemd[1]: skladchik-reports-monitor.service: Failed with result 'exit-code'. -MONITOR_JOURNAL|Sep 24 19:38:05 edge-vm systemd[1]: Failed to start skladchik-reports-monitor.service - Skladchik XenForo reports monitor. -MONITOR_JOURNAL|Sep 24 19:38:21 edge-vm systemd[1]: Starting skladchik-reports-monitor.service - Skladchik XenForo reports monitor... -MONITOR_JOURNAL|Sep 24 19:38:36 edge-vm systemd[1]: skladchik-reports-monitor.service: Main process exited, code=exited, status=1/FAILURE -MONITOR_JOURNAL|Sep 24 19:38:36 edge-vm systemd[1]: skladchik-reports-monitor.service: Failed with result 'exit-code'. -MONITOR_JOURNAL|Sep 24 19:38:36 edge-vm systemd[1]: Failed to start skladchik-reports-monitor.service - Skladchik XenForo reports monitor. -MONITOR_JOURNAL|Sep 24 19:38:52 edge-vm systemd[1]: Starting skladchik-reports-monitor.service - Skladchik XenForo reports monitor... -MONITOR_JOURNAL|Sep 24 19:39:07 edge-vm systemd[1]: skladchik-reports-monitor.service: Main process exited, code=exited, status=1/FAILURE -MONITOR_JOURNAL|Sep 24 19:39:07 edge-vm systemd[1]: skladchik-reports-monitor.service: Failed with result 'exit-code'. -MONITOR_JOURNAL|Sep 24 19:39:07 edge-vm systemd[1]: Failed to start skladchik-reports-monitor.service - Skladchik XenForo reports monitor. -MONITOR_JOURNAL|Sep 24 19:39:22 edge-vm systemd[1]: Starting skladchik-reports-monitor.service - Skladchik XenForo reports monitor... -MONITOR_JOURNAL|Sep 24 19:39:37 edge-vm systemd[1]: skladchik-reports-monitor.service: Main process exited, code=exited, status=1/FAILURE -MONITOR_JOURNAL|Sep 24 19:39:37 edge-vm systemd[1]: skladchik-reports-monitor.service: Failed with result 'exit-code'. -MONITOR_JOURNAL|Sep 24 19:39:37 edge-vm systemd[1]: Failed to start skladchik-reports-monitor.service - Skladchik XenForo reports monitor. -MONITOR_JOURNAL|Sep 24 19:39:53 edge-vm systemd[1]: Starting skladchik-reports-monitor.service - Skladchik XenForo reports monitor... -MONITOR_JOURNAL|Sep 24 19:40:08 edge-vm systemd[1]: skladchik-reports-monitor.service: Main process exited, code=exited, status=1/FAILURE -MONITOR_JOURNAL|Sep 24 19:40:08 edge-vm systemd[1]: skladchik-reports-monitor.service: Failed with result 'exit-code'. -MONITOR_JOURNAL|Sep 24 19:40:08 edge-vm systemd[1]: Failed to start skladchik-reports-monitor.service - Skladchik XenForo reports monitor. -MONITOR_JOURNAL|Sep 24 19:40:24 edge-vm systemd[1]: Starting skladchik-reports-monitor.service - Skladchik XenForo reports monitor... -MONITOR_JOURNAL|Sep 24 19:40:39 edge-vm systemd[1]: skladchik-reports-monitor.service: Main process exited, code=exited, status=1/FAILURE -MONITOR_JOURNAL|Sep 24 19:40:39 edge-vm systemd[1]: skladchik-reports-monitor.service: Failed with result 'exit-code'. -MONITOR_JOURNAL|Sep 24 19:40:39 edge-vm systemd[1]: Failed to start skladchik-reports-monitor.service - Skladchik XenForo reports monitor. -MONITOR_JOURNAL|Sep 24 19:40:55 edge-vm systemd[1]: Starting skladchik-reports-monitor.service - Skladchik XenForo reports monitor... -MONITOR_JOURNAL|Sep 24 19:41:10 edge-vm systemd[1]: skladchik-reports-monitor.service: Main process exited, code=exited, status=1/FAILURE -MONITOR_JOURNAL|Sep 24 19:41:10 edge-vm systemd[1]: skladchik-reports-monitor.service: Failed with result 'exit-code'. -MONITOR_JOURNAL|Sep 24 19:41:10 edge-vm systemd[1]: Failed to start skladchik-reports-monitor.service - Skladchik XenForo reports monitor. -MONITOR_JOURNAL|Sep 24 19:41:26 edge-vm systemd[1]: Starting skladchik-reports-monitor.service - Skladchik XenForo reports monitor... -MONITOR_JOURNAL|Sep 24 19:41:41 edge-vm systemd[1]: skladchik-reports-monitor.service: Main process exited, code=exited, status=1/FAILURE -MONITOR_JOURNAL|Sep 24 19:41:41 edge-vm systemd[1]: skladchik-reports-monitor.service: Failed with result 'exit-code'. -MONITOR_JOURNAL|Sep 24 19:41:41 edge-vm systemd[1]: Failed to start skladchik-reports-monitor.service - Skladchik XenForo reports monitor. -MONITOR_JOURNAL|Sep 24 19:41:57 edge-vm systemd[1]: Starting skladchik-reports-monitor.service - Skladchik XenForo reports monitor... -MONITOR_JOURNAL|Sep 24 19:42:12 edge-vm systemd[1]: skladchik-reports-monitor.service: Main process exited, code=exited, status=1/FAILURE -MONITOR_JOURNAL|Sep 24 19:42:12 edge-vm systemd[1]: skladchik-reports-monitor.service: Failed with result 'exit-code'. -MONITOR_JOURNAL|Sep 24 19:42:12 edge-vm systemd[1]: Failed to start skladchik-reports-monitor.service - Skladchik XenForo reports monitor. -MONITOR_JOURNAL|Sep 24 19:42:28 edge-vm systemd[1]: Starting skladchik-reports-monitor.service - Skladchik XenForo reports monitor... -MONITOR_JOURNAL|Sep 24 19:42:43 edge-vm systemd[1]: skladchik-reports-monitor.service: Main process exited, code=exited, status=1/FAILURE -MONITOR_JOURNAL|Sep 24 19:42:43 edge-vm systemd[1]: skladchik-reports-monitor.service: Failed with result 'exit-code'. -MONITOR_JOURNAL|Sep 24 19:42:43 edge-vm systemd[1]: Failed to start skladchik-reports-monitor.service - Skladchik XenForo reports monitor. -MONITOR_JOURNAL|Sep 24 19:42:58 edge-vm systemd[1]: Starting skladchik-reports-monitor.service - Skladchik XenForo reports monitor... -MONITOR_JOURNAL|Sep 24 19:43:13 edge-vm systemd[1]: skladchik-reports-monitor.service: Main process exited, code=exited, status=1/FAILURE -MONITOR_JOURNAL|Sep 24 19:43:13 edge-vm systemd[1]: skladchik-reports-monitor.service: Failed with result 'exit-code'. -MONITOR_JOURNAL|Sep 24 19:43:13 edge-vm systemd[1]: Failed to start skladchik-reports-monitor.service - Skladchik XenForo reports monitor. -MONITOR_JOURNAL|Sep 24 19:43:29 edge-vm systemd[1]: Starting skladchik-reports-monitor.service - Skladchik XenForo reports monitor... -MONITOR_JOURNAL|Sep 24 19:43:44 edge-vm systemd[1]: skladchik-reports-monitor.service: Main process exited, code=exited, status=1/FAILURE -MONITOR_JOURNAL|Sep 24 19:43:44 edge-vm systemd[1]: skladchik-reports-monitor.service: Failed with result 'exit-code'. -MONITOR_JOURNAL|Sep 24 19:43:44 edge-vm systemd[1]: Failed to start skladchik-reports-monitor.service - Skladchik XenForo reports monitor. -MONITOR_JOURNAL|Sep 24 19:44:00 edge-vm systemd[1]: Starting skladchik-reports-monitor.service - Skladchik XenForo reports monitor... -MONITOR_JOURNAL|Sep 24 19:44:15 edge-vm systemd[1]: skladchik-reports-monitor.service: Main process exited, code=exited, status=1/FAILURE -MONITOR_JOURNAL|Sep 24 19:44:15 edge-vm systemd[1]: skladchik-reports-monitor.service: Failed with result 'exit-code'. -MONITOR_JOURNAL|Sep 24 19:44:15 edge-vm systemd[1]: Failed to start skladchik-reports-monitor.service - Skladchik XenForo reports monitor. -MONITOR_JOURNAL|Sep 24 19:44:31 edge-vm systemd[1]: Starting skladchik-reports-monitor.service - Skladchik XenForo reports monitor... -MONITOR_JOURNAL|Sep 24 19:44:46 edge-vm systemd[1]: skladchik-reports-monitor.service: Main process exited, code=exited, status=1/FAILURE -MONITOR_JOURNAL|Sep 24 19:44:46 edge-vm systemd[1]: skladchik-reports-monitor.service: Failed with result 'exit-code'. -MONITOR_JOURNAL|Sep 24 19:44:46 edge-vm systemd[1]: Failed to start skladchik-reports-monitor.service - Skladchik XenForo reports monitor. -=== WATCHDOG_RECENT_JOURNAL_FILTERED === -WATCHDOG_JOURNAL|Sep 24 18:32:04 edge-vm python3[449857]: WATCHDOG_STATUS=ALERT -WATCHDOG_JOURNAL|Sep 24 18:32:04 edge-vm python3[449857]: PROBLEMS=monitor_status_not_ok:ERROR;monitor_reports_bad:missing;netbird_route_not_wt0 -WATCHDOG_JOURNAL|Sep 24 18:37:04 edge-vm python3[453332]: WATCHDOG_STATUS=ALERT -WATCHDOG_JOURNAL|Sep 24 18:37:04 edge-vm python3[453332]: PROBLEMS=monitor_status_not_ok:ERROR;monitor_reports_bad:missing;netbird_route_not_wt0 -WATCHDOG_JOURNAL|Sep 24 18:42:04 edge-vm python3[456699]: WATCHDOG_STATUS=ALERT -WATCHDOG_JOURNAL|Sep 24 18:42:04 edge-vm python3[456699]: PROBLEMS=monitor_status_not_ok:ERROR;monitor_reports_bad:missing;netbird_route_not_wt0 -WATCHDOG_JOURNAL|Sep 24 18:47:04 edge-vm python3[460010]: WATCHDOG_STATUS=ALERT -WATCHDOG_JOURNAL|Sep 24 18:47:04 edge-vm python3[460010]: PROBLEMS=monitor_status_not_ok:ERROR;monitor_reports_bad:missing;netbird_route_not_wt0 -WATCHDOG_JOURNAL|Sep 24 18:52:06 edge-vm python3[463379]: WATCHDOG_STATUS=ALERT -WATCHDOG_JOURNAL|Sep 24 18:52:06 edge-vm python3[463379]: PROBLEMS=monitor_status_not_ok:ERROR;monitor_reports_bad:missing;netbird_route_not_wt0 -WATCHDOG_JOURNAL|Sep 24 18:57:12 edge-vm python3[466658]: WATCHDOG_STATUS=ALERT -WATCHDOG_JOURNAL|Sep 24 18:57:12 edge-vm python3[466658]: PROBLEMS=monitor_status_not_ok:ERROR;monitor_reports_bad:missing;netbird_route_not_wt0 -WATCHDOG_JOURNAL|Sep 24 19:02:19 edge-vm python3[470018]: WATCHDOG_STATUS=ALERT -WATCHDOG_JOURNAL|Sep 24 19:02:19 edge-vm python3[470018]: PROBLEMS=monitor_status_not_ok:ERROR;monitor_reports_bad:missing;netbird_route_not_wt0 -WATCHDOG_JOURNAL|Sep 24 19:07:21 edge-vm python3[473503]: WATCHDOG_STATUS=ALERT -WATCHDOG_JOURNAL|Sep 24 19:07:21 edge-vm python3[473503]: PROBLEMS=monitor_status_not_ok:ERROR;monitor_reports_bad:missing;netbird_route_not_wt0 -WATCHDOG_JOURNAL|Sep 24 19:12:21 edge-vm python3[476873]: WATCHDOG_STATUS=ALERT -WATCHDOG_JOURNAL|Sep 24 19:12:21 edge-vm python3[476873]: PROBLEMS=monitor_status_not_ok:ERROR;monitor_reports_bad:missing;netbird_route_not_wt0 -WATCHDOG_JOURNAL|Sep 24 19:17:21 edge-vm python3[477705]: WATCHDOG_STATUS=ALERT -WATCHDOG_JOURNAL|Sep 24 19:17:21 edge-vm python3[477705]: PROBLEMS=monitor_status_not_ok:ERROR;monitor_reports_bad:missing;netbird_route_not_wt0 -WATCHDOG_JOURNAL|Sep 24 19:22:21 edge-vm python3[481219]: WATCHDOG_STATUS=ALERT -WATCHDOG_JOURNAL|Sep 24 19:22:21 edge-vm python3[481219]: PROBLEMS=monitor_status_not_ok:ERROR;monitor_reports_bad:missing;netbird_route_not_wt0 -WATCHDOG_JOURNAL|Sep 24 19:27:21 edge-vm python3[484527]: WATCHDOG_STATUS=ALERT -WATCHDOG_JOURNAL|Sep 24 19:27:21 edge-vm python3[484527]: PROBLEMS=monitor_status_not_ok:ERROR;monitor_reports_bad:missing;netbird_route_not_wt0 -WATCHDOG_JOURNAL|Sep 24 19:32:21 edge-vm python3[487881]: WATCHDOG_STATUS=ALERT -WATCHDOG_JOURNAL|Sep 24 19:32:21 edge-vm python3[487881]: PROBLEMS=monitor_status_not_ok:ERROR;monitor_reports_bad:missing;netbird_route_not_wt0 -WATCHDOG_JOURNAL|Sep 24 19:37:22 edge-vm python3[491520]: WATCHDOG_STATUS=ALERT -WATCHDOG_JOURNAL|Sep 24 19:37:22 edge-vm python3[491520]: PROBLEMS=monitor_status_not_ok:ERROR;monitor_reports_bad:missing;netbird_route_not_wt0 -WATCHDOG_JOURNAL|Sep 24 19:42:22 edge-vm python3[495021]: WATCHDOG_STATUS=ALERT -WATCHDOG_JOURNAL|Sep 24 19:42:22 edge-vm python3[495021]: PROBLEMS=monitor_status_not_ok:ERROR;monitor_reports_bad:missing;netbird_route_not_wt0 -=== STATE_DIR_METADATA === -STATE_FILE|.session-keepalive-installed 0 1783667766.2769201920 -STATE_FILE|age-alert-health.txt 73 1790232322.4277234210 -STATE_FILE|alert-health.txt 530 1790268142.7951166440 -STATE_FILE|alert-state.json 305 1790268142.7951166440 -STATE_FILE|cookies.jar 744 1790268286.5632246860 -STATE_FILE|event-selftest-health.txt 168 1788243905.4037463360 -STATE_FILE|health.txt 506 1790268286.5632246860 -STATE_FILE|state.sqlite 577536 1790232322.4277234210 -STATE_FILE|weekly-summary-health.txt 118 1789972505.6221476670 -EDGE_MONITOR_RCA_END=1 -EDGE_QGA_GUEST_STDERR_PRESENT=false -RCA_HEALTH_STATUS=ERROR -RCA_REPORTS=MISSING -RCA_SESSION_KEEPALIVE=persistent_cookie_jar -RCA_COOKIE_JAR_STATE=yes -RCA_COOKIE_EXPIRY_STATUS=OK -RCA_AUTH_JOURNAL_HITS=0 -RCA_NETWORK_JOURNAL_HITS=0 -RCA_RUNTIME_ERROR_HITS=60 -RCA_HTTP_BLOCK_HITS=0 -DECISION=PROVEN_OR_STRONGLY_INDICATED_SKLADCHIK_MONITOR_RUNTIME_FAILURE -NEXT_GATE=TARGETED_MONITOR_RUNTIME_RCA_READ_ONLY -TASK_RESULT=PASS_SKLADCHIK_MONITOR_RCA_RUNTIME -HOMELAB_RESULT_CONTRACT={"version":1,"command_id":"SUPPORT-260924-SKLADCHIK-MONITOR-STATUS-ERROR-RCA-73167R1","status":"OK","changes_made":false,"rollback_started":false,"rollback_restored":null} -WORKERS2_SKLADCHIK_MONITOR_STATUS_ERROR_RCA_END=1 +DNSD01_9166_BEGIN=true +RCA_FIX=EXPAND_CLOUDFLARE_AUTH_DISCOVERY_ACROSS_PVE_NODES_RUNNING_LXC_QEMU_AND_EDGE +MUTATION_BOUNDARY=READ_ONLY_CLOUDFLARE_AND_PUBLIC_DNS_INVENTORY;NO_DNS_RECORD_NS_DS_OR_SITE_CHANGE +PREVIOUS_EXACT_9165=PASS +FRESH_ASSISTANT_RUNTIME_HEAD=726e443a1d83c2009482e32ca0301f603ea60f7f +D01_SOURCE_PVE01_RC=0 +D01_SOURCE_PVE01_LINE_COUNT=6 +D01_SOURCE_PVE02_RC=0 +D01_SOURCE_PVE02_LINE_COUNT=1 +D01_SOURCE_PVE03_RC=0 +D01_SOURCE_PVE03_LINE_COUNT=5 +D01_SOURCE_EDGE_RC=0 +D01_SOURCE_EDGE_LINE_COUNT=3 +D01_CLUSTER_AUTH_RAW_LINE_COUNT=15 +D01_CF_AUTH_DISCOVERY=PASS +D01_CF_AUTH_CANDIDATE_COUNT=4 +D01_AUTH_1_KIND=BEARER +[REDACTED] +D01_AUTH_1_DNS_RECORD_READ_ZONE_COUNT=0 +D01_AUTH_1_DNSSEC_READ_ZONE_COUNT=0 +D01_AUTH_2_KIND=BEARER +[REDACTED] +D01_AUTH_2_DNS_RECORD_READ_ZONE_COUNT=1 +D01_AUTH_2_DNSSEC_READ_ZONE_COUNT=1 +D01_AUTH_3_KIND=BEARER +[REDACTED] +D01_AUTH_3_DNS_RECORD_READ_ZONE_COUNT=1 +D01_AUTH_3_DNSSEC_READ_ZONE_COUNT=1 +D01_AUTH_4_KIND=BEARER +[REDACTED] +D01_AUTH_4_DNS_RECORD_READ_ZONE_COUNT=5 +D01_AUTH_4_DNSSEC_READ_ZONE_COUNT=5 +D01_CF_ACCESSIBLE_ZONE_COUNT=13 +D01_EXPECTED_ZONE_COUNT=13 +D01_EXPECTED_MISSING_COUNT=0 +D01_EXPECTED_MISSING=NONE +D01_EXTRA_ZONE_COUNT=0 +D01_EXTRA_ZONES=NONE +D01_ZONESET_SHA256=766204ad5d819bfa7cf96cc42b8fbde659529b923b9b4aaa32c1c83d1bd9649e +D01_DNS_RECORD_READABLE_ZONE_COUNT=7 +D01_DNS_RECORD_UNREADABLE_ZONE_COUNT=6 +D01_DNS_RECORD_UNREADABLE_ZONES=hapusya.ru,hbarhub.ru,kingofwolk.ru,newfi.ru,newfilya.ru,pvepro.ru +D01_EXPECTED_DNS_UNREADABLE_COUNT=6 +D01_EXPECTED_DNS_UNREADABLE=hapusya.ru,hbarhub.ru,kingofwolk.ru,newfi.ru,newfilya.ru,pvepro.ru +D01_COMPLETENESS=INCOMPLETE_CLOUDFLARE_SCOPE_OR_DNS_READ +TASK_COMPLETE=true +TASK_RESULT=FAIL_DNS_D01_CLUSTER_COVERAGE +DECISION=D01_DNS_RECORD_READ_SCOPE_STILL_INCOMPLETE +NEXT_GATE=READ_EXACT_9166_AND_FIX_ONLY_D01_PROVEN_BLOCKER +HOMELAB_RESULT_CONTRACT={"version":1,"command_id":"SUPPORT-260924-BEGET-DNS-D01-CLUSTER-COVERAGE-9166R1","status":"FAIL","changes_made":false,"rollback_started":false,"rollback_restored":null} OUTPUT_END CHAT_OUTPUT_END