homelab-admin/homelab-public-context
1
0
Fork 0
Code Activity

runtime: publish SUPPORT-261006-PVE01-REMAINING-FAILED-UNITS-RCA-99694R1

Browse Source
This commit is contained in:
homelab-runtime-publisher
2026-10-06 08:36:42 +00:00
parent 99353de4e8
commit 588c23c032
4 changed files with 509 additions and 94 deletions
Download Patch File Download Diff File Expand all files Collapse all files
runtime/history/SUPPORT-261006-PVE01-REMAINING-FAILED-UNITS-RCA-99694R1.json
+38
View File
File diff suppressed because one or more lines are too long
runtime/history/SUPPORT-261006-PVE01-REMAINING-FAILED-UNITS-RCA-99694R1.txt
+243
View File
@@ -0,0 +1,243 @@
CHAT_OUTPUT_BEGIN
COMMAND_ID=SUPPORT-261006-PVE01-REMAINING-FAILED-UNITS-RCA-99694R1
STATUS=OK
RC=0
HOST=pve01
MODE=read-only
COMPONENT=pve01-remaining-failed-units-rca
REFERENCE_REGISTER_CHECK=OK
REFERENCE_SHA256=5a3d8e5154c41cb582a4e0aca68090be1f0138918bf82131a948df326f9d8d66
ERROR_REGISTER_CHECK=OK
ERROR_REGISTER_SHA256=3b09a553ec0f527ed3afeed4753f52a74ee3036045bbfb3c685e9f8af4ba7ba0
COMMAND_SHA256=18e8f9dc2347e65c7d6310e27f48bdd48e3a1978f434af5fee6ab8ab48185dac
DUPLICATE_FAILED_COMMAND_BLOCKED=false
EXECUTION_STARTED=true
CHANGE_DECLARED=false
RESULT_CONTRACT_VALID=true
RESULT_CONTRACT_STATUS=NOT_APPLICABLE
RESULT_CONTRACT_ERROR=NONE
COMMAND_RC=0
CHANGES_MADE=false
ROLLBACK_STARTED=false
ROLLBACK_RESTORED=null
MUTATION_OUTCOME=NO_MUTATION
SANITIZED=yes
SECRETS_INCLUDED=no
PRIVATE_ADDRESSES_INCLUDED=no
RAW_EVIDENCE_SHA256=4228b00b8b8cab02f55eb05e3b5c238c8acdf45fcc4c59e66c5978d7c6ee6fd1
SANITIZED_OUTPUT_SHA256=4228b00b8b8cab02f55eb05e3b5c238c8acdf45fcc4c59e66c5978d7c6ee6fd1
OUTPUT_BEGIN
PVE01_REMAINING_FAILED_UNITS_RCA=V1
MODE=read-only
NO_REBOOT=true
NO_APT_UPDATE=true
NO_PACKAGE_MUTATION=true
NO_SERVICE_START_STOP_RESTART=true
NO_RESET_FAILED=true
NO_CONFIG_WRITE=true
NO_FILE_DELETE=true
NO_GUEST_MUTATION=true
NO_BACKUP_EXECUTION=true
NO_NETWORK_MUTATION=true
NODE_HOSTNAME=pve01
UNIFIED_SHA=c588c4aa0c941406efcb3d24876c9262d9ec0dd4ced297d456fd2193e83afc49
V2_SHA=7f413e4a3bc276be668ad0d7a877a0f2b2bd64bf1dead9cb929d966b25a44e25
HELPER_SHA=66a243a207cbc5e19255fcafdb06013008d1d8c1d1205e660fdf0f236ff6a15a
PROBE_SHA=44ea82521afb37e862ec7fb7f0e3eeb2f0f582b784506d8dec23f73ccea86050
WORKER_SHA=a86bc027eefc37bbbcb9ad9b18c1c2c2a4f4ef9172a9cd9918a00f1d94c9e19a
HELPER_VALIDATE_RC=0
CANONICAL_PROBE_RC=0
CANONICAL_PROBE_SEALED_PROVEN=true
BACKUP_AUDIT_EXEC_PRESENT=true
BACKUP_AUDIT_EXEC_BASENAME=homelab-backup-audit
BACKUP_AUDIT_EXEC_SHA=1c2cd8a00859492c010ed35baf5eb67f4278544809de8c2563421a6f3e356ed3
BACKUP_AUDIT_RESULT_NOW=exit-code
BACKUP_AUDIT_EXEC_STATUS_NOW=1
BACKUP_AUDIT_FRESH_LINES=48
REC|AUDIT_FRESH|TARGET|logical_id=app-filebrowser|kind=APP|state=FRESH|legacy_age_seconds=14441|generation=app-filebrowser-pve01-20261006T043010Z
REC|AUDIT_FRESH|TARGET|logical_id=app-hbarhub|kind=APP|state=FRESH|legacy_age_seconds=14279|generation=app-hbarhub-pve01-20261006T043148Z
REC|AUDIT_FRESH|TARGET|logical_id=app-immich|kind=APP|state=FRESH|legacy_age_seconds=14385|generation=app-immich-pve01-20261006T043031Z
REC|AUDIT_FRESH|TARGET|logical_id=app-skladchik-reports-monitor|kind=APP|state=FRESH|legacy_age_seconds=43247|generation=app-skladchik-reports-monitor-pve01-20261005T203005Z
REC|AUDIT_FRESH|TARGET|logical_id=control-bootstrap-v2|kind=CONTROL|state=FRESH|age_seconds=370|generation=control-bootstrap-v2-pve01-20260916T120057Z
REC|AUDIT_FRESH|TARGET|logical_id=vm110|kind=VM|state=FRESH|source=V2|v2_age_seconds=3875|legacy_age_seconds=2775090|replica_count=2|state_id=ct110|generation=v2-ct110-20261006T072242Z-655b3dc1c042
REC|AUDIT_FRESH|TARGET|logical_id=vm111|kind=VM|state=FRESH|source=V2|v2_age_seconds=1522|replica_count=2|state_id=ct111|generation=v2-ct111-20261006T080259Z-417f9545ef1b
REC|AUDIT_FRESH|TARGET|logical_id=vm112|kind=VM|state=FRESH|source=V2|v2_age_seconds=1|legacy_age_seconds=2813585|replica_count=2|state_id=ct112|generation=v2-ct112-20261006T082950Z-2538063ab2f8
REC|AUDIT_FRESH|TARGET|logical_id=vm113|kind=VM|state=FRESH|source=V2|v2_age_seconds=67118|replica_count=2|state_id=ct113|generation=v2-ct113-20261005T135058Z-4ef751f00af7
REC|AUDIT_FRESH|TARGET|logical_id=vm130|kind=VM|state=FRESH|source=V2|v2_age_seconds=5102|replica_count=2|state_id=vm130|generation=v2-vm130-20261006T064416Z-6541a84254d2
REC|AUDIT_FRESH|TARGET|logical_id=vm150|kind=VM|state=FRESH|source=V2|v2_age_seconds=65014|legacy_age_seconds=83432|replica_count=2|state_id=vm150|generation=v2-vm150-20261005T142532Z-b95b67f5ddb0
REC|AUDIT_FRESH|TARGET|logical_id=vm170|kind=VM|state=FRESH|source=V2|v2_age_seconds=51585|legacy_age_seconds=1719827|replica_count=2|state_id=vm170|generation=v2-vm170-20261005T175552Z-16895f2ebc94
REC|AUDIT_FRESH|TARGET|logical_id=vm171|kind=VM|state=FRESH|source=V2|v2_age_seconds=47820|legacy_age_seconds=1749359|replica_count=2|state_id=vm171|generation=v2-vm171-20261005T191016Z-0bc9caa6fc1d
REC|AUDIT_FRESH|TARGET|logical_id=vm190|kind=VM|state=FRESH|source=V2|v2_age_seconds=62823|replica_count=2|state_id=vm190|generation=v2-vm190-20261005T150007Z-12d7245298b9
REC|AUDIT_FRESH|TARGET|logical_id=vm9130|kind=VM|state=FRESH|source=V2|v2_age_seconds=30496|replica_count=2|state_id=vm9130|generation=v2-vm9130-20261005T234232Z-c716cfa66ce9
REC|AUDIT_FRESH|ISSUE|severity=CRITICAL|code=APP_FRESHNESS_PLAN_FAILED|logical_id=-|root_cause=APP_QUEUE_RC_65
REC|AUDIT_FRESH|ISSUE|severity=CRITICAL|code=GFS_INVALID|logical_id=-|root_cause=GFS_VALIDATE_FAILED
REC|AUDIT_FRESH|ISSUE|severity=HIGH|code=RETENTION_CONTRACT_BAD|logical_id=-|root_cause=RETENTION_CONTRACT_RC_65
REC|AUDIT_FRESH|SUMMARY|AUDIT_STATUS=OK|BACKUP_STATUS=RED|ACTIVE_COUNT=15|ISSUE_COUNT=3|CLOUD_A=OK|CLOUD_B=OK|CLOUD_B_DRILL=OK|DR_PROVEN=YES|SOURCE_OFFHOST=OK
REC|AUDIT_FRESH|TARGET|logical_id=app-filebrowser|kind=APP|state=FRESH|legacy_age_seconds=14739|generation=app-filebrowser-pve01-20261006T043010Z
REC|AUDIT_FRESH|TARGET|logical_id=app-hbarhub|kind=APP|state=FRESH|legacy_age_seconds=14577|generation=app-hbarhub-pve01-20261006T043148Z
REC|AUDIT_FRESH|TARGET|logical_id=app-immich|kind=APP|state=FRESH|legacy_age_seconds=14683|generation=app-immich-pve01-20261006T043031Z
REC|AUDIT_FRESH|TARGET|logical_id=app-skladchik-reports-monitor|kind=APP|state=FRESH|legacy_age_seconds=43545|generation=app-skladchik-reports-monitor-pve01-20261005T203005Z
REC|AUDIT_FRESH|TARGET|logical_id=control-bootstrap-v2|kind=CONTROL|state=FRESH|age_seconds=668|generation=control-bootstrap-v2-pve01-20260916T120057Z
REC|AUDIT_FRESH|TARGET|logical_id=vm110|kind=VM|state=FRESH|source=V2|v2_age_seconds=4173|legacy_age_seconds=2775388|replica_count=2|state_id=ct110|generation=v2-ct110-20261006T072242Z-655b3dc1c042
REC|AUDIT_FRESH|TARGET|logical_id=vm111|kind=VM|state=FRESH|source=V2|v2_age_seconds=1820|replica_count=2|state_id=ct111|generation=v2-ct111-20261006T080259Z-417f9545ef1b
REC|AUDIT_FRESH|TARGET|logical_id=vm112|kind=VM|state=FRESH|source=V2|v2_age_seconds=299|legacy_age_seconds=2813883|replica_count=2|state_id=ct112|generation=v2-ct112-20261006T082950Z-2538063ab2f8
REC|AUDIT_FRESH|TARGET|logical_id=vm113|kind=VM|state=FRESH|source=V2|v2_age_seconds=67416|replica_count=2|state_id=ct113|generation=v2-ct113-20261005T135058Z-4ef751f00af7
REC|AUDIT_FRESH|TARGET|logical_id=vm130|kind=VM|state=FRESH|source=V2|v2_age_seconds=5400|replica_count=2|state_id=vm130|generation=v2-vm130-20261006T064416Z-6541a84254d2
REC|AUDIT_FRESH|TARGET|logical_id=vm150|kind=VM|state=FRESH|source=V2|v2_age_seconds=65312|legacy_age_seconds=83730|replica_count=2|state_id=vm150|generation=v2-vm150-20261005T142532Z-b95b67f5ddb0
REC|AUDIT_FRESH|TARGET|logical_id=vm170|kind=VM|state=FRESH|source=V2|v2_age_seconds=51883|legacy_age_seconds=1720125|replica_count=2|state_id=vm170|generation=v2-vm170-20261005T175552Z-16895f2ebc94
REC|AUDIT_FRESH|TARGET|logical_id=vm171|kind=VM|state=FRESH|source=V2|v2_age_seconds=48118|legacy_age_seconds=1749657|replica_count=2|state_id=vm171|generation=v2-vm171-20261005T191016Z-0bc9caa6fc1d
REC|AUDIT_FRESH|TARGET|logical_id=vm190|kind=VM|state=FRESH|source=V2|v2_age_seconds=63121|replica_count=2|state_id=vm190|generation=v2-vm190-20261005T150007Z-12d7245298b9
REC|AUDIT_FRESH|TARGET|logical_id=vm9130|kind=VM|state=FRESH|source=V2|v2_age_seconds=30794|replica_count=2|state_id=vm9130|generation=v2-vm9130-20261005T234232Z-c716cfa66ce9
REC|AUDIT_FRESH|ISSUE|severity=CRITICAL|code=APP_FRESHNESS_PLAN_FAILED|logical_id=-|root_cause=APP_QUEUE_RC_65
REC|AUDIT_FRESH|ISSUE|severity=CRITICAL|code=GFS_INVALID|logical_id=-|root_cause=GFS_VALIDATE_FAILED
REC|AUDIT_FRESH|ISSUE|severity=HIGH|code=RETENTION_CONTRACT_BAD|logical_id=-|root_cause=RETENTION_CONTRACT_RC_65
REC|AUDIT_FRESH|SUMMARY|AUDIT_STATUS=OK|BACKUP_STATUS=RED|ACTIVE_COUNT=15|ISSUE_COUNT=3|CLOUD_A=OK|CLOUD_B=OK|CLOUD_B_DRILL=OK|DR_PROVEN=YES|SOURCE_OFFHOST=OK
AUDIT_SOURCE_APP_FRESHNESS_PLAN_FAILED_HIT_COUNT=1
REC|AUDIT_SOURCE|CODE=APP_FRESHNESS_PLAN_FAILED|LINE=112|resources={}
REC|AUDIT_SOURCE|CODE=APP_FRESHNESS_PLAN_FAILED|LINE=113|if rrc==0:
REC|AUDIT_SOURCE|CODE=APP_FRESHNESS_PLAN_FAILED|LINE=114|try:
REC|AUDIT_SOURCE|CODE=APP_FRESHNESS_PLAN_FAILED|LINE=115|for x in json.loads(rout): resources[str(x.get("vmid"))]=x
REC|AUDIT_SOURCE|CODE=APP_FRESHNESS_PLAN_FAILED|LINE=116|except Exception: pass
REC|AUDIT_SOURCE|CODE=APP_FRESHNESS_PLAN_FAILED|LINE=117|
REC|AUDIT_SOURCE|CODE=APP_FRESHNESS_PLAN_FAILED|LINE=118|qrc,qout,qerr=run([APPQ,"freshness-plan"],180)
REC|AUDIT_SOURCE|CODE=APP_FRESHNESS_PLAN_FAILED|LINE=119|app_dec=parse_app_plan(qout if qrc==0 else "")
REC|AUDIT_SOURCE|CODE=APP_FRESHNESS_PLAN_FAILED|LINE=120|if qrc!=0: issues.append({"severity":"CRITICAL","code":"APP_FRESHNESS_PLAN_FAILED","logical_id":"-","root_cause":"APP_QUEUE_RC_"+str(qrc)})
REC|AUDIT_SOURCE|CODE=APP_FRESHNESS_PLAN_FAILED|LINE=121|
REC|AUDIT_SOURCE|CODE=APP_FRESHNESS_PLAN_FAILED|LINE=122|for lid,r in sorted(active.items()):
REC|AUDIT_SOURCE|CODE=APP_FRESHNESS_PLAN_FAILED|LINE=123|kind=r["kind"]; ident=r["id"]; rec={"logical_id":lid,"kind":kind,"state":"UNVERIFIED"}
REC|AUDIT_SOURCE|CODE=APP_FRESHNESS_PLAN_FAILED|LINE=124|if kind=="VM":
REC|AUDIT_SOURCE|CODE=APP_FRESHNESS_PLAN_FAILED|LINE=125|typ=str(resources.get(ident,{}).get("type","qemu"))
REC|AUDIT_SOURCE|CODE=APP_FRESHNESS_PLAN_FAILED|LINE=126|sid=("ct"+ident) if typ=="lxc" else ("vm"+ident)
REC|AUDIT_SOURCE|CODE=APP_FRESHNESS_PLAN_FAILED|LINE=127|v2=None
REC|AUDIT_SOURCE|CODE=APP_FRESHNESS_PLAN_FAILED|LINE=128|try: v2=json.loads((STATE/(sid+".json")).read_text(encoding="utf-8"))
AUDIT_SOURCE_GFS_INVALID_HIT_COUNT=1
REC|AUDIT_SOURCE|CODE=GFS_INVALID|LINE=207|if lock=="PATH_REDACTED and expected_v2_heavy_lock_owner():
REC|AUDIT_SOURCE|CODE=GFS_INVALID|LINE=208|pass
REC|AUDIT_SOURCE|CODE=GFS_INVALID|LINE=209|else:
REC|AUDIT_SOURCE|CODE=GFS_INVALID|LINE=210|issues.append({"severity":"HIGH","code":"LOCK_BUSY","logical_id":lock,"root_cause":"ACTIVE_OR_ORPHAN_LOCK"})
REC|AUDIT_SOURCE|CODE=GFS_INVALID|LINE=211|
REC|AUDIT_SOURCE|CODE=GFS_INVALID|LINE=212|env=os.environ.copy(); env["HOMELAB_BACKUP_GFS_POLICY"]=GFSP; env["HOMELAB_BACKUP_TARGET_REGISTRY"]=str(REG)
REC|AUDIT_SOURCE|CODE=GFS_INVALID|LINE=213|grc,gout,_=run([GFSH,"validate"],20,env)
REC|AUDIT_SOURCE|CODE=GFS_INVALID|LINE=214|if grc!=0 or "GFS_POLICY=OK" not in gout:
REC|AUDIT_SOURCE|CODE=GFS_INVALID|LINE=215|issues.append({"severity":"CRITICAL","code":"GFS_INVALID","logical_id":"-","root_cause":"GFS_VALIDATE_FAILED"})
REC|AUDIT_SOURCE|CODE=GFS_INVALID|LINE=216|rr,_,_=run([RET,"contract"],120)
REC|AUDIT_SOURCE|CODE=GFS_INVALID|LINE=217|if rr!=0: issues.append({"severity":"HIGH","code":"RETENTION_CONTRACT_BAD","logical_id":"-","root_cause":"RETENTION_CONTRACT_RC_"+str(rr)})
REC|AUDIT_SOURCE|CODE=GFS_INVALID|LINE=218|rhp=pathlib.Path("PATH_REDACTED
REC|AUDIT_SOURCE|CODE=GFS_INVALID|LINE=219|if not rhp.is_file():
REC|AUDIT_SOURCE|CODE=GFS_INVALID|LINE=220|issues.append({"severity":"HIGH","code":"RETENTION_HEALTH_MISSING","logical_id":"-","root_cause":"RETENTION_HEALTH_FILE_MISSING"})
REC|AUDIT_SOURCE|CODE=GFS_INVALID|LINE=221|else:
REC|AUDIT_SOURCE|CODE=GFS_INVALID|LINE=222|try:
REC|AUDIT_SOURCE|CODE=GFS_INVALID|LINE=223|rh=json.loads(rhp.read_text(encoding="utf-8"))
AUDIT_SOURCE_RETENTION_CONTRACT_BAD_HIT_COUNT=1
REC|AUDIT_SOURCE|CODE=RETENTION_CONTRACT_BAD|LINE=209|else:
REC|AUDIT_SOURCE|CODE=RETENTION_CONTRACT_BAD|LINE=210|issues.append({"severity":"HIGH","code":"LOCK_BUSY","logical_id":lock,"root_cause":"ACTIVE_OR_ORPHAN_LOCK"})
REC|AUDIT_SOURCE|CODE=RETENTION_CONTRACT_BAD|LINE=211|
REC|AUDIT_SOURCE|CODE=RETENTION_CONTRACT_BAD|LINE=212|env=os.environ.copy(); env["HOMELAB_BACKUP_GFS_POLICY"]=GFSP; env["HOMELAB_BACKUP_TARGET_REGISTRY"]=str(REG)
REC|AUDIT_SOURCE|CODE=RETENTION_CONTRACT_BAD|LINE=213|grc,gout,_=run([GFSH,"validate"],20,env)
REC|AUDIT_SOURCE|CODE=RETENTION_CONTRACT_BAD|LINE=214|if grc!=0 or "GFS_POLICY=OK" not in gout:
REC|AUDIT_SOURCE|CODE=RETENTION_CONTRACT_BAD|LINE=215|issues.append({"severity":"CRITICAL","code":"GFS_INVALID","logical_id":"-","root_cause":"GFS_VALIDATE_FAILED"})
REC|AUDIT_SOURCE|CODE=RETENTION_CONTRACT_BAD|LINE=216|rr,_,_=run([RET,"contract"],120)
REC|AUDIT_SOURCE|CODE=RETENTION_CONTRACT_BAD|LINE=217|if rr!=0: issues.append({"severity":"HIGH","code":"RETENTION_CONTRACT_BAD","logical_id":"-","root_cause":"RETENTION_CONTRACT_RC_"+str(rr)})
REC|AUDIT_SOURCE|CODE=RETENTION_CONTRACT_BAD|LINE=218|rhp=pathlib.Path("PATH_REDACTED
REC|AUDIT_SOURCE|CODE=RETENTION_CONTRACT_BAD|LINE=219|if not rhp.is_file():
REC|AUDIT_SOURCE|CODE=RETENTION_CONTRACT_BAD|LINE=220|issues.append({"severity":"HIGH","code":"RETENTION_HEALTH_MISSING","logical_id":"-","root_cause":"RETENTION_HEALTH_FILE_MISSING"})
REC|AUDIT_SOURCE|CODE=RETENTION_CONTRACT_BAD|LINE=221|else:
REC|AUDIT_SOURCE|CODE=RETENTION_CONTRACT_BAD|LINE=222|try:
REC|AUDIT_SOURCE|CODE=RETENTION_CONTRACT_BAD|LINE=223|rh=json.loads(rhp.read_text(encoding="utf-8"))
REC|AUDIT_SOURCE|CODE=RETENTION_CONTRACT_BAD|LINE=224|if rh.get("status")!="OK": issues.append({"severity":"HIGH","code":"RETENTION_HEALTH_BAD","logical_id":"-","root_cause":"RETENTION_HEALTH_NOT_OK"})
REC|AUDIT_SOURCE|CODE=RETENTION_CONTRACT_BAD|LINE=225|except Exception: issues.append({"severity":"HIGH","code":"RETENTION_HEALTH_BAD","logical_id":"-","root_cause":"RETENTION_HEALTH_UNREADABLE"})
REC|AUDIT_PATH_REF|BASENAME=targets.json|PARENT=homelab-backup|TYPE=file|SHA256=c588c4aa0c941406efcb3d24876c9262d9ec0dd4ced297d456fd2193e83afc49|PATH_SHA=2f97838a94d9466163297d47d72c9c2e903a12cfb344abc58ff33fcbbae05528
REC|AUDIT_PATH_REF|BASENAME=state|PARENT=homelab-backup-v2|TYPE=dir|SHA256=NONE|PATH_SHA=6b3144b23b95313197d83e8d55533ff96cc06062af4ae20c3d7a635cdc56145e
REC|AUDIT_PATH_REF|BASENAME=cloud-quorum|PARENT=homelab-backup|TYPE=dir|SHA256=NONE|PATH_SHA=8757ada57c0d5e9d3c7d43ff9f4463f9cd391a8f19c6952b0c872e10a305630c
REC|AUDIT_PATH_REF|BASENAME=vm-restore-validation|PARENT=homelab-backup|TYPE=dir|SHA256=NONE|PATH_SHA=c56996e4ea53994a88601ceeed1e5291c3e7ea79661386090cf582c3036616f8
REC|AUDIT_PATH_REF|BASENAME=app-restore-validation|PARENT=homelab-backup|TYPE=dir|SHA256=NONE|PATH_SHA=c70a6d9e128e0f6fc7e29453ab78510eb2ff57124d19767bd88f87659063751c
REC|AUDIT_PATH_REF|BASENAME=homelab-backup-target-registry|PARENT=libexec|TYPE=file|SHA256=66a243a207cbc5e19255fcafdb06013008d1d8c1d1205e660fdf0f236ff6a15a|PATH_SHA=cd6e91b27939aee60ecf8f93a42b3c793926bfd8930593f3ae3bb95f72692162
REC|AUDIT_PATH_REF|BASENAME=homelab-backup-gfs-policy|PARENT=libexec|TYPE=file|SHA256=a1aed0c6039f8095bd5da7f62b47f4ee6932df98c4e0e7420d3006df86bd5c94|PATH_SHA=d731f7b73d3dd86c74980bed86852aea70b61074102fc4792f958096a43441a9
REC|AUDIT_PATH_REF|BASENAME=gfs-retention.json|PARENT=homelab-backup|TYPE=file|SHA256=2c7674045dd5c68cc48d348d616494dbbe145df460a8e85d97a2c5c153ec792d|PATH_SHA=e76524b2cd2eff155d83a860d2ff2c65c0b54695eb6445ecc2af88cfbf32816d
REC|AUDIT_PATH_REF|BASENAME=homelab-backup-retention-gc|PARENT=sbin|TYPE=file|SHA256=a96e607107e7b4e7762f7cb7f0e91f8c3ec6b793ed4604633c125433839ac909|PATH_SHA=68bcf5e97e4eda63f88bfee3ece5bc50ccdcbb2a089f97923534e67daf29ee4a
REC|AUDIT_PATH_REF|BASENAME=restore-check-health.txt|PARENT=skladchik-reports-monitor-backup|TYPE=file|SHA256=08a290bea388e4ca0e1160aab1c909ff0cef9e98326b6dd32abe9f5541125ce8|PATH_SHA=c7b01e883c50df0f85e999374fa9d383932f47207b7a088028bbbd3fae148509
REC|AUDIT_PATH_REF|BASENAME=backup-audit-v2.json|PARENT=homelab-health|TYPE=file|SHA256=2ece5343c5d732cbaa65d16feafaade791a2eb2eae465d8a6aba4e644c12e49a|PATH_SHA=e520cdf4c0a133ac68165ccf96f4bec1031f4d7a3e7bddccf5c84aea2a26cad7
REC|AUDIT_PATH_REF|BASENAME=backup-audit.json|PARENT=homelab-health|TYPE=file|SHA256=2ece5343c5d732cbaa65d16feafaade791a2eb2eae465d8a6aba4e644c12e49a|PATH_SHA=091a828db6ee335d0272a370019f274a88f3a0fa4c9fd92d268358a3674f0b1d
REC|AUDIT_PATH_REF|BASENAME=homelab-backup-heavy.lock|PARENT=run|TYPE=file|SHA256=e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855|PATH_SHA=2642241b0beaa7376e1362beefa000e2137478b93e8c1a5d275189b4954a6c93
REC|AUDIT_PATH_REF|BASENAME=homelab-backup-heavy.lock|PARENT=run|TYPE=file|SHA256=e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855|PATH_SHA=2642241b0beaa7376e1362beefa000e2137478b93e8c1a5d275189b4954a6c93
REC|AUDIT_PATH_REF|BASENAME=homelab-backup-dispatcher.lock|PARENT=lock|TYPE=file|SHA256=e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855|PATH_SHA=448d0e8fe73ac64db817c9df04aede92c9cea524652755a9fcdf60fbad00c1aa
REC|AUDIT_PATH_REF|BASENAME=homelab-backup-heavy.lock|PARENT=run|TYPE=file|SHA256=e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855|PATH_SHA=2642241b0beaa7376e1362beefa000e2137478b93e8c1a5d275189b4954a6c93
REC|AUDIT_PATH_REF|BASENAME=backup-retention-safe-gc.json|PARENT=homelab-health|TYPE=file|SHA256=4260fcbc5797375b9281e9c64fec35b203fcd35110395e10b94a602a2a67b48c|PATH_SHA=9427dedc6832eded4f955d7a1b6251430298b78c17ed2970664df3370485c719
REC|AUDIT_PATH_REF|BASENAME=app-skladchik-reports-monitor|PARENT=cloud-quorum|TYPE=dir|SHA256=NONE|PATH_SHA=dc50a937c87b10fe436df9da05826c17b7a3d17d316d87035e5571051c8358a3
REC|AUDIT_PATH_REF|BASENAME=app-skladchik-reports-monitor|PARENT=app-restore-validation|TYPE=dir|SHA256=NONE|PATH_SHA=c75d5957fa9c11e592017a8399ae56adcdb992ade172a54162a8177c1291f45a
BACKUP_AUDIT_DIRECT_RC=1
REC|AUDIT_DIRECT|ISSUE|severity=CRITICAL|code=APP_FRESHNESS_PLAN_FAILED|logical_id=-|root_cause=APP_QUEUE_RC_65
REC|AUDIT_DIRECT|ISSUE|severity=CRITICAL|code=GFS_INVALID|logical_id=-|root_cause=GFS_VALIDATE_FAILED
REC|AUDIT_DIRECT|ISSUE|severity=HIGH|code=RETENTION_CONTRACT_BAD|logical_id=-|root_cause=RETENTION_CONTRACT_RC_65
REC|AUDIT_DIRECT|SUMMARY|AUDIT_STATUS=OK|BACKUP_STATUS=RED|ACTIVE_COUNT=15|ISSUE_COUNT=3|CLOUD_A=OK|CLOUD_B=OK|CLOUD_B_DRILL=OK|DR_PROVEN=YES|SOURCE_OFFHOST=OK
BACKUP_AUDIT_DIRECT_STDERR_LINES=0
VPN_EXEC_PRESENT=true
VPN_EXEC_BASENAME=bash
VPN_EXEC_SHA=223fe8564b60636bc738b6178d3ba9a50ef7d791266b0efae6363bb716e4c47f
VPN_RESULT_NOW=exit-code
VPN_EXEC_STATUS_NOW=1
REC|VPN_JOURNAL|1|Starting homelab-private-vpn-hosts-health.service - Homelab private VPN hosts health check...
REC|VPN_JOURNAL|2|homelab-private-vpn-hosts-health.service: Main process exited, code=exited, status=1/FAILURE
REC|VPN_JOURNAL|3|homelab-private-vpn-hosts-health.service: Failed with result 'exit-code'.
REC|VPN_JOURNAL|4|Failed to start homelab-private-vpn-hosts-health.service - Homelab private VPN hosts health check.
REC|VPN_JOURNAL|5|Starting homelab-private-vpn-hosts-health.service - Homelab private VPN hosts health check...
REC|VPN_JOURNAL|6|homelab-private-vpn-hosts-health.service: Main process exited, code=exited, status=1/FAILURE
REC|VPN_JOURNAL|7|homelab-private-vpn-hosts-health.service: Failed with result 'exit-code'.
REC|VPN_JOURNAL|8|Failed to start homelab-private-vpn-hosts-health.service - Homelab private VPN hosts health check.
REC|VPN_JOURNAL|9|Starting homelab-private-vpn-hosts-health.service - Homelab private VPN hosts health check...
REC|VPN_JOURNAL|10|homelab-private-vpn-hosts-health.service: Main process exited, code=exited, status=1/FAILURE
REC|VPN_JOURNAL|11|homelab-private-vpn-hosts-health.service: Failed with result 'exit-code'.
REC|VPN_JOURNAL|12|Failed to start homelab-private-vpn-hosts-health.service - Homelab private VPN hosts health check.
REC|VPN_JOURNAL|13|Starting homelab-private-vpn-hosts-health.service - Homelab private VPN hosts health check...
REC|VPN_JOURNAL|14|homelab-private-vpn-hosts-health.service: Main process exited, code=exited, status=1/FAILURE
REC|VPN_JOURNAL|15|homelab-private-vpn-hosts-health.service: Failed with result 'exit-code'.
REC|VPN_JOURNAL|16|Failed to start homelab-private-vpn-hosts-health.service - Homelab private VPN hosts health check.
REC|VPN_JOURNAL|17|Starting homelab-private-vpn-hosts-health.service - Homelab private VPN hosts health check...
REC|VPN_JOURNAL|18|homelab-private-vpn-hosts-health.service: Main process exited, code=exited, status=1/FAILURE
REC|VPN_JOURNAL|19|homelab-private-vpn-hosts-health.service: Failed with result 'exit-code'.
REC|VPN_JOURNAL|20|Failed to start homelab-private-vpn-hosts-health.service - Homelab private VPN hosts health check.
REC|VPN_JOURNAL|21|Starting homelab-private-vpn-hosts-health.service - Homelab private VPN hosts health check...
REC|VPN_JOURNAL|22|homelab-private-vpn-hosts-health.service: Main process exited, code=exited, status=1/FAILURE
REC|VPN_JOURNAL|23|homelab-private-vpn-hosts-health.service: Failed with result 'exit-code'.
REC|VPN_JOURNAL|24|Failed to start homelab-private-vpn-hosts-health.service - Homelab private VPN hosts health check.
REC|VPN_JOURNAL|25|Starting homelab-private-vpn-hosts-health.service - Homelab private VPN hosts health check...
REC|VPN_JOURNAL|26|homelab-private-vpn-hosts-health.service: Main process exited, code=exited, status=1/FAILURE
REC|VPN_JOURNAL|27|homelab-private-vpn-hosts-health.service: Failed with result 'exit-code'.
REC|VPN_JOURNAL|28|Failed to start homelab-private-vpn-hosts-health.service - Homelab private VPN hosts health check.
REC|VPN_JOURNAL|29|Starting homelab-private-vpn-hosts-health.service - Homelab private VPN hosts health check...
REC|VPN_JOURNAL|30|homelab-private-vpn-hosts-health.service: Main process exited, code=exited, status=1/FAILURE
REC|VPN_JOURNAL|31|homelab-private-vpn-hosts-health.service: Failed with result 'exit-code'.
REC|VPN_JOURNAL|32|Failed to start homelab-private-vpn-hosts-health.service - Homelab private VPN hosts health check.
REC|VPN_JOURNAL|33|Starting homelab-private-vpn-hosts-health.service - Homelab private VPN hosts health check...
REC|VPN_JOURNAL|34|homelab-private-vpn-hosts-health.service: Main process exited, code=exited, status=1/FAILURE
REC|VPN_JOURNAL|35|homelab-private-vpn-hosts-health.service: Failed with result 'exit-code'.
REC|VPN_JOURNAL|36|Failed to start homelab-private-vpn-hosts-health.service - Homelab private VPN hosts health check.
REC|VPN_JOURNAL|37|Starting homelab-private-vpn-hosts-health.service - Homelab private VPN hosts health check...
REC|VPN_JOURNAL|38|homelab-private-vpn-hosts-health.service: Main process exited, code=exited, status=1/FAILURE
REC|VPN_JOURNAL|39|homelab-private-vpn-hosts-health.service: Failed with result 'exit-code'.
REC|VPN_JOURNAL|40|Failed to start homelab-private-vpn-hosts-health.service - Homelab private VPN hosts health check.
VPN_EXPECTED_NAME_COUNT=0
VPN_PVE01_RESOLV_CONF_SHA=a89871ac2938a423c74c6dfe801eab57085b4a4c869bb1f864097f5885e27a74
VPN_PVE01_RESOLV_NAMESERVER_COUNT=1
VPN_DIRECT_RC=0
REC|VPN_DIRECT_OUT|1|PVE2_VPN_UNIT_EXISTS=false
REC|VPN_DIRECT_OUT|2|PVE2_VPN_RESULT=success
REC|VPN_DIRECT_OUT|3|PVE2_RESOLV_SHA=466da4d9381fbd060c635244fa97a800e50b77613fbd8c2ea0d2057ad4617f1f
REC|VPN_DIRECT_OUT|4|PVE2_RESOLV_NAMESERVER_COUNT=1
REC|VPN_DIRECT_OUT|5|FAILED_UNIT_COUNT_NOW=2
REC|VPN_DIRECT_OUT|6|REC|FAILED_UNIT_NOW|homelab-backup-audit.service
REC|VPN_DIRECT_OUT|7|REC|FAILED_UNIT_NOW|homelab-private-vpn-hosts-health.service
REC|VPN_DIRECT_OUT|8|APT_FULL_UPGRADE_SIM_RC=0
REC|VPN_DIRECT_OUT|9|APT_FULL_UPGRADE_SIM_UPGRADE_COUNT=184
REC|VPN_DIRECT_OUT|10|APT_FULL_UPGRADE_SIM_REMOVE_COUNT=0
REC|VPN_DIRECT_OUT|11|DECISION=PASS_PVE01_REMAINING_FAILED_UNITS_RCA
REC|VPN_DIRECT_OUT|12|NEXT_GATE=PVE01_REMAINING_FAILED_UNITS_REPAIR
REC|VPN_DIRECT_OUT|13|TASK_RESULT=PVE01_REMAINING_FAILED_UNITS_RCA
REC|VPN_DIRECT_OUT|14|CHANGES_MADE=false
REC|VPN_DIRECT_OUT|15|ROLLBACK_STARTED=false
REC|VPN_DIRECT_OUT|16|ROLLBACK_RESTORED=null
REC|VPN_DIRECT_OUT|17|MUTATION_OUTCOME=NO_MUTATION
REC|VPN_DIRECT_OUT|18|HOMELAB_RESULT_CONTRACT={"version":1,"command_id":"SUPPORT-261006-PVE01-REMAINING-FAILED-UNITS-RCA-99694R1","status":"OK","changes_made":false,"rollback_started":false,"rollback_restored":null}
OUTPUT_END
CHAT_OUTPUT_END
runtime/latest.json
+13 -13
View File
File diff suppressed because one or more lines are too long
runtime/latest.txt
+215 -81
View File
@@ -1,109 +1,243 @@
CHAT_OUTPUT_BEGIN
COMMAND_ID=SUPPORT-261006-PVE01-BACKUP-V2-CT112-RECOVERY-99693R1
COMMAND_ID=SUPPORT-261006-PVE01-REMAINING-FAILED-UNITS-RCA-99694R1
STATUS=OK
RC=0
HOST=pve01
MODE=change
COMPONENT=pve01-backup-v2-ct112-recovery
MODE=read-only
COMPONENT=pve01-remaining-failed-units-rca
REFERENCE_REGISTER_CHECK=OK
REFERENCE_SHA256=5a3d8e5154c41cb582a4e0aca68090be1f0138918bf82131a948df326f9d8d66
ERROR_REGISTER_CHECK=OK
ERROR_REGISTER_SHA256=3b09a553ec0f527ed3afeed4753f52a74ee3036045bbfb3c685e9f8af4ba7ba0
COMMAND_SHA256=c5cbf57fc30df1fee72fd9d03ba994d049714a2fb662aff01897e435c6b4308e
COMMAND_SHA256=18e8f9dc2347e65c7d6310e27f48bdd48e3a1978f434af5fee6ab8ab48185dac
DUPLICATE_FAILED_COMMAND_BLOCKED=false
EXECUTION_STARTED=true
CHANGE_DECLARED=true
CHANGE_DECLARED=false
RESULT_CONTRACT_VALID=true
RESULT_CONTRACT_STATUS=OK
RESULT_CONTRACT_STATUS=NOT_APPLICABLE
RESULT_CONTRACT_ERROR=NONE
COMMAND_RC=0
CHANGES_MADE=true
CHANGES_MADE=false
ROLLBACK_STARTED=false
ROLLBACK_RESTORED=null
MUTATION_OUTCOME=MUTATED
MUTATION_OUTCOME=NO_MUTATION
SANITIZED=yes
SECRETS_INCLUDED=no
PRIVATE_ADDRESSES_INCLUDED=no
RAW_EVIDENCE_SHA256=6c2bf4b4508d61650cfd16dc40e874e6fb2907a5e72aa95001723c9d8e00e2ec
SANITIZED_OUTPUT_SHA256=6c2bf4b4508d61650cfd16dc40e874e6fb2907a5e72aa95001723c9d8e00e2ec
RAW_EVIDENCE_SHA256=4228b00b8b8cab02f55eb05e3b5c238c8acdf45fcc4c59e66c5978d7c6ee6fd1
SANITIZED_OUTPUT_SHA256=4228b00b8b8cab02f55eb05e3b5c238c8acdf45fcc4c59e66c5978d7c6ee6fd1
OUTPUT_BEGIN
PVE01_BACKUP_V2_CT112_RECOVERY=V1
MODE=change
SCOPE=ONE_REAL_SCHEDULER_CYCLE_CT112_AND_FINAL_RPO_AUDIT
PVE01_REMAINING_FAILED_UNITS_RCA=V1
MODE=read-only
NO_REBOOT=true
NO_APT_UPDATE=true
NO_PACKAGE_MUTATION=true
NO_SERVICE_START_STOP_RESTART=true
NO_RESET_FAILED=true
NO_CONFIG_WRITE=true
NO_FILE_DELETE=true
NO_GUEST_CONFIG_MUTATION=true
BACKUP_EXECUTION=true
BACKUP_CYCLE_LIMIT=1
NO_NETWORK_CONFIG_MUTATION=true
NO_VPN_SERVICE_EXECUTION=true
NO_GUEST_MUTATION=true
NO_BACKUP_EXECUTION=true
NO_NETWORK_MUTATION=true
NODE_HOSTNAME=pve01
REC|SHA_PRE|BASENAME=targets.json|SHA=c588c4aa0c941406efcb3d24876c9262d9ec0dd4ced297d456fd2193e83afc49
REC|SHA_PRE|BASENAME=targets.json|SHA=7f413e4a3bc276be668ad0d7a877a0f2b2bd64bf1dead9cb929d966b25a44e25
REC|SHA_PRE|BASENAME=homelab-backup-target-registry|SHA=66a243a207cbc5e19255fcafdb06013008d1d8c1d1205e660fdf0f236ff6a15a
REC|SHA_PRE|BASENAME=homelab-app-cloud-quorum-freshness-probe|SHA=44ea82521afb37e862ec7fb7f0e3eeb2f0f582b784506d8dec23f73ccea86050
REC|SHA_PRE|BASENAME=homelab-app-cloud-quorum-worker|SHA=a86bc027eefc37bbbcb9ad9b18c1c2c2a4f4ef9172a9cd9918a00f1d94c9e19a
V2_VM211_COUNT=1
V2_VM211_DISABLED=true
V2_DISABLED_TOTAL=11
DUE_TARGET_COUNT_PRE=2
REC|DUE_BY_DEADLINE|RANK=1|INDEX=2|LOGICAL_ID=ct112|TYPE=pve_guest|VMID=112|LAST=1791193606|DUE_SECONDS=72000|DUE_AT=1791265606
REC|DUE_BY_DEADLINE|RANK=2|INDEX=13|LOGICAL_ID=node-pve01|TYPE=pve_node_bundle|VMID=|LAST=1790662089|DUE_SECONDS=604800|DUE_AT=1791266889
EXPECTED_NEXT_TARGET_PRE=ct112
EXPECTED_NEXT_VMID_PRE=112
EXPECTED_NEXT_LIVE_MATCH_COUNT_PRE=1
REC|EXPECTED_NEXT_LIVE|VMID=112|TYPE=lxc|NODE=pve01|STATUS=running
CT112_STATE_PRESENT_PRE=true
CT112_LAST_SUCCESS_PRE=1791193606
CT112_LAST_ATTEMPT_PRE=1791193606
CT112_RESULT_PRE=SUCCESS
CT112_GENERATION_PRE=v2-ct112-20261005T094532Z-ee250335d44e
BACKUP_V2_ACTIVE_PRE=inactive
BACKUP_V2_START_RC=0
BACKUP_V2_RESULT_POST=success
BACKUP_V2_EXEC_STATUS_POST=0
BACKUP_V2_FRESH_JOURNAL_LINES=14
REC|BACKUP_V2_CURRENT|SCHEDULE_TARGET=ct112
REC|BACKUP_V2_CURRENT|SCHEDULE_RESULT=SUCCESS
ACTUAL_SCHEDULE_TARGET=ct112
CT112_STATE_PRESENT_POST=true
CT112_LAST_SUCCESS_POST=1791275459
CT112_LAST_SUCCESS_ADVANCED=true
CT112_RESULT_POST=SUCCESS
CT112_REASON_POST=MISSING
CT112_GENERATION_POST=v2-ct112-20261006T082950Z-2538063ab2f8
Job for homelab-backup-audit.service failed because the control process exited with error code.
See "systemctl status homelab-backup-audit.service" and "journalctl -xeu homelab-backup-audit.service" for details.
BACKUP_AUDIT_START_RC=1
BACKUP_AUDIT_EXEC_STATUS_POST=1
BACKUP_AUDIT_ISSUE_COUNT_POST=3
BACKUP_AUDIT_RPO_BREACH_COUNT_POST=0
BACKUP_AUDIT_RPO_BREACH_IDS_POST=
BACKUP_AUDIT_VM112_ISSUE_COUNT_POST=0
BACKUP_AUDIT_SUMMARY_AUDIT_STATUS=OK
BACKUP_AUDIT_SUMMARY_BACKUP_STATUS=RED
BACKUP_AUDIT_SUMMARY_ISSUE_COUNT=3
BACKUP_AUDIT_SUMMARY_ACTIVE_COUNT=15
REC|AUDIT_ISSUE_POST|LOGICAL_ID=-|CODE=APP_FRESHNESS_PLAN_FAILED
REC|AUDIT_ISSUE_POST|LOGICAL_ID=-|CODE=GFS_INVALID
REC|AUDIT_ISSUE_POST|LOGICAL_ID=-|CODE=RETENTION_CONTRACT_BAD
FAILED_UNIT_COUNT_POST=2
REC|FAILED_UNIT_POST|homelab-backup-audit.service
REC|FAILED_UNIT_POST|homelab-private-vpn-hosts-health.service
APT_FULL_UPGRADE_SIM_RC=0
APT_FULL_UPGRADE_SIM_UPGRADE_COUNT=184
APT_FULL_UPGRADE_SIM_REMOVE_COUNT=0
DECISION=PASS_PVE01_BACKUP_V2_CT112_RECOVERY
NEXT_GATE=PVE01_VPN_HEALTH_REPAIR
TASK_RESULT=PVE01_BACKUP_V2_CT112_RECOVERY
CHANGES_MADE=true
ROLLBACK_STARTED=false
ROLLBACK_RESTORED=null
MUTATION_OUTCOME=CT112_RECOVERY_AND_ZERO_RPO_COMPLETED
HOMELAB_RESULT_CONTRACT={"version":1,"command_id":"SUPPORT-261006-PVE01-BACKUP-V2-CT112-RECOVERY-99693R1","status":"OK","changes_made":true,"rollback_started":false,"rollback_restored":null}
UNIFIED_SHA=c588c4aa0c941406efcb3d24876c9262d9ec0dd4ced297d456fd2193e83afc49
V2_SHA=7f413e4a3bc276be668ad0d7a877a0f2b2bd64bf1dead9cb929d966b25a44e25
HELPER_SHA=66a243a207cbc5e19255fcafdb06013008d1d8c1d1205e660fdf0f236ff6a15a
PROBE_SHA=44ea82521afb37e862ec7fb7f0e3eeb2f0f582b784506d8dec23f73ccea86050
WORKER_SHA=a86bc027eefc37bbbcb9ad9b18c1c2c2a4f4ef9172a9cd9918a00f1d94c9e19a
HELPER_VALIDATE_RC=0
CANONICAL_PROBE_RC=0
CANONICAL_PROBE_SEALED_PROVEN=true
BACKUP_AUDIT_EXEC_PRESENT=true
BACKUP_AUDIT_EXEC_BASENAME=homelab-backup-audit
BACKUP_AUDIT_EXEC_SHA=1c2cd8a00859492c010ed35baf5eb67f4278544809de8c2563421a6f3e356ed3
BACKUP_AUDIT_RESULT_NOW=exit-code
BACKUP_AUDIT_EXEC_STATUS_NOW=1
BACKUP_AUDIT_FRESH_LINES=48
REC|AUDIT_FRESH|TARGET|logical_id=app-filebrowser|kind=APP|state=FRESH|legacy_age_seconds=14441|generation=app-filebrowser-pve01-20261006T043010Z
REC|AUDIT_FRESH|TARGET|logical_id=app-hbarhub|kind=APP|state=FRESH|legacy_age_seconds=14279|generation=app-hbarhub-pve01-20261006T043148Z
REC|AUDIT_FRESH|TARGET|logical_id=app-immich|kind=APP|state=FRESH|legacy_age_seconds=14385|generation=app-immich-pve01-20261006T043031Z
REC|AUDIT_FRESH|TARGET|logical_id=app-skladchik-reports-monitor|kind=APP|state=FRESH|legacy_age_seconds=43247|generation=app-skladchik-reports-monitor-pve01-20261005T203005Z
REC|AUDIT_FRESH|TARGET|logical_id=control-bootstrap-v2|kind=CONTROL|state=FRESH|age_seconds=370|generation=control-bootstrap-v2-pve01-20260916T120057Z
REC|AUDIT_FRESH|TARGET|logical_id=vm110|kind=VM|state=FRESH|source=V2|v2_age_seconds=3875|legacy_age_seconds=2775090|replica_count=2|state_id=ct110|generation=v2-ct110-20261006T072242Z-655b3dc1c042
REC|AUDIT_FRESH|TARGET|logical_id=vm111|kind=VM|state=FRESH|source=V2|v2_age_seconds=1522|replica_count=2|state_id=ct111|generation=v2-ct111-20261006T080259Z-417f9545ef1b
REC|AUDIT_FRESH|TARGET|logical_id=vm112|kind=VM|state=FRESH|source=V2|v2_age_seconds=1|legacy_age_seconds=2813585|replica_count=2|state_id=ct112|generation=v2-ct112-20261006T082950Z-2538063ab2f8
REC|AUDIT_FRESH|TARGET|logical_id=vm113|kind=VM|state=FRESH|source=V2|v2_age_seconds=67118|replica_count=2|state_id=ct113|generation=v2-ct113-20261005T135058Z-4ef751f00af7
REC|AUDIT_FRESH|TARGET|logical_id=vm130|kind=VM|state=FRESH|source=V2|v2_age_seconds=5102|replica_count=2|state_id=vm130|generation=v2-vm130-20261006T064416Z-6541a84254d2
REC|AUDIT_FRESH|TARGET|logical_id=vm150|kind=VM|state=FRESH|source=V2|v2_age_seconds=65014|legacy_age_seconds=83432|replica_count=2|state_id=vm150|generation=v2-vm150-20261005T142532Z-b95b67f5ddb0
REC|AUDIT_FRESH|TARGET|logical_id=vm170|kind=VM|state=FRESH|source=V2|v2_age_seconds=51585|legacy_age_seconds=1719827|replica_count=2|state_id=vm170|generation=v2-vm170-20261005T175552Z-16895f2ebc94
REC|AUDIT_FRESH|TARGET|logical_id=vm171|kind=VM|state=FRESH|source=V2|v2_age_seconds=47820|legacy_age_seconds=1749359|replica_count=2|state_id=vm171|generation=v2-vm171-20261005T191016Z-0bc9caa6fc1d
REC|AUDIT_FRESH|TARGET|logical_id=vm190|kind=VM|state=FRESH|source=V2|v2_age_seconds=62823|replica_count=2|state_id=vm190|generation=v2-vm190-20261005T150007Z-12d7245298b9
REC|AUDIT_FRESH|TARGET|logical_id=vm9130|kind=VM|state=FRESH|source=V2|v2_age_seconds=30496|replica_count=2|state_id=vm9130|generation=v2-vm9130-20261005T234232Z-c716cfa66ce9
REC|AUDIT_FRESH|ISSUE|severity=CRITICAL|code=APP_FRESHNESS_PLAN_FAILED|logical_id=-|root_cause=APP_QUEUE_RC_65
REC|AUDIT_FRESH|ISSUE|severity=CRITICAL|code=GFS_INVALID|logical_id=-|root_cause=GFS_VALIDATE_FAILED
REC|AUDIT_FRESH|ISSUE|severity=HIGH|code=RETENTION_CONTRACT_BAD|logical_id=-|root_cause=RETENTION_CONTRACT_RC_65
REC|AUDIT_FRESH|SUMMARY|AUDIT_STATUS=OK|BACKUP_STATUS=RED|ACTIVE_COUNT=15|ISSUE_COUNT=3|CLOUD_A=OK|CLOUD_B=OK|CLOUD_B_DRILL=OK|DR_PROVEN=YES|SOURCE_OFFHOST=OK
REC|AUDIT_FRESH|TARGET|logical_id=app-filebrowser|kind=APP|state=FRESH|legacy_age_seconds=14739|generation=app-filebrowser-pve01-20261006T043010Z
REC|AUDIT_FRESH|TARGET|logical_id=app-hbarhub|kind=APP|state=FRESH|legacy_age_seconds=14577|generation=app-hbarhub-pve01-20261006T043148Z
REC|AUDIT_FRESH|TARGET|logical_id=app-immich|kind=APP|state=FRESH|legacy_age_seconds=14683|generation=app-immich-pve01-20261006T043031Z
REC|AUDIT_FRESH|TARGET|logical_id=app-skladchik-reports-monitor|kind=APP|state=FRESH|legacy_age_seconds=43545|generation=app-skladchik-reports-monitor-pve01-20261005T203005Z
REC|AUDIT_FRESH|TARGET|logical_id=control-bootstrap-v2|kind=CONTROL|state=FRESH|age_seconds=668|generation=control-bootstrap-v2-pve01-20260916T120057Z
REC|AUDIT_FRESH|TARGET|logical_id=vm110|kind=VM|state=FRESH|source=V2|v2_age_seconds=4173|legacy_age_seconds=2775388|replica_count=2|state_id=ct110|generation=v2-ct110-20261006T072242Z-655b3dc1c042
REC|AUDIT_FRESH|TARGET|logical_id=vm111|kind=VM|state=FRESH|source=V2|v2_age_seconds=1820|replica_count=2|state_id=ct111|generation=v2-ct111-20261006T080259Z-417f9545ef1b
REC|AUDIT_FRESH|TARGET|logical_id=vm112|kind=VM|state=FRESH|source=V2|v2_age_seconds=299|legacy_age_seconds=2813883|replica_count=2|state_id=ct112|generation=v2-ct112-20261006T082950Z-2538063ab2f8
REC|AUDIT_FRESH|TARGET|logical_id=vm113|kind=VM|state=FRESH|source=V2|v2_age_seconds=67416|replica_count=2|state_id=ct113|generation=v2-ct113-20261005T135058Z-4ef751f00af7
REC|AUDIT_FRESH|TARGET|logical_id=vm130|kind=VM|state=FRESH|source=V2|v2_age_seconds=5400|replica_count=2|state_id=vm130|generation=v2-vm130-20261006T064416Z-6541a84254d2
REC|AUDIT_FRESH|TARGET|logical_id=vm150|kind=VM|state=FRESH|source=V2|v2_age_seconds=65312|legacy_age_seconds=83730|replica_count=2|state_id=vm150|generation=v2-vm150-20261005T142532Z-b95b67f5ddb0
REC|AUDIT_FRESH|TARGET|logical_id=vm170|kind=VM|state=FRESH|source=V2|v2_age_seconds=51883|legacy_age_seconds=1720125|replica_count=2|state_id=vm170|generation=v2-vm170-20261005T175552Z-16895f2ebc94
REC|AUDIT_FRESH|TARGET|logical_id=vm171|kind=VM|state=FRESH|source=V2|v2_age_seconds=48118|legacy_age_seconds=1749657|replica_count=2|state_id=vm171|generation=v2-vm171-20261005T191016Z-0bc9caa6fc1d
REC|AUDIT_FRESH|TARGET|logical_id=vm190|kind=VM|state=FRESH|source=V2|v2_age_seconds=63121|replica_count=2|state_id=vm190|generation=v2-vm190-20261005T150007Z-12d7245298b9
REC|AUDIT_FRESH|TARGET|logical_id=vm9130|kind=VM|state=FRESH|source=V2|v2_age_seconds=30794|replica_count=2|state_id=vm9130|generation=v2-vm9130-20261005T234232Z-c716cfa66ce9
REC|AUDIT_FRESH|ISSUE|severity=CRITICAL|code=APP_FRESHNESS_PLAN_FAILED|logical_id=-|root_cause=APP_QUEUE_RC_65
REC|AUDIT_FRESH|ISSUE|severity=CRITICAL|code=GFS_INVALID|logical_id=-|root_cause=GFS_VALIDATE_FAILED
REC|AUDIT_FRESH|ISSUE|severity=HIGH|code=RETENTION_CONTRACT_BAD|logical_id=-|root_cause=RETENTION_CONTRACT_RC_65
REC|AUDIT_FRESH|SUMMARY|AUDIT_STATUS=OK|BACKUP_STATUS=RED|ACTIVE_COUNT=15|ISSUE_COUNT=3|CLOUD_A=OK|CLOUD_B=OK|CLOUD_B_DRILL=OK|DR_PROVEN=YES|SOURCE_OFFHOST=OK
AUDIT_SOURCE_APP_FRESHNESS_PLAN_FAILED_HIT_COUNT=1
REC|AUDIT_SOURCE|CODE=APP_FRESHNESS_PLAN_FAILED|LINE=112|resources={}
REC|AUDIT_SOURCE|CODE=APP_FRESHNESS_PLAN_FAILED|LINE=113|if rrc==0:
REC|AUDIT_SOURCE|CODE=APP_FRESHNESS_PLAN_FAILED|LINE=114|try:
REC|AUDIT_SOURCE|CODE=APP_FRESHNESS_PLAN_FAILED|LINE=115|for x in json.loads(rout): resources[str(x.get("vmid"))]=x
REC|AUDIT_SOURCE|CODE=APP_FRESHNESS_PLAN_FAILED|LINE=116|except Exception: pass
REC|AUDIT_SOURCE|CODE=APP_FRESHNESS_PLAN_FAILED|LINE=117|
REC|AUDIT_SOURCE|CODE=APP_FRESHNESS_PLAN_FAILED|LINE=118|qrc,qout,qerr=run([APPQ,"freshness-plan"],180)
REC|AUDIT_SOURCE|CODE=APP_FRESHNESS_PLAN_FAILED|LINE=119|app_dec=parse_app_plan(qout if qrc==0 else "")
REC|AUDIT_SOURCE|CODE=APP_FRESHNESS_PLAN_FAILED|LINE=120|if qrc!=0: issues.append({"severity":"CRITICAL","code":"APP_FRESHNESS_PLAN_FAILED","logical_id":"-","root_cause":"APP_QUEUE_RC_"+str(qrc)})
REC|AUDIT_SOURCE|CODE=APP_FRESHNESS_PLAN_FAILED|LINE=121|
REC|AUDIT_SOURCE|CODE=APP_FRESHNESS_PLAN_FAILED|LINE=122|for lid,r in sorted(active.items()):
REC|AUDIT_SOURCE|CODE=APP_FRESHNESS_PLAN_FAILED|LINE=123|kind=r["kind"]; ident=r["id"]; rec={"logical_id":lid,"kind":kind,"state":"UNVERIFIED"}
REC|AUDIT_SOURCE|CODE=APP_FRESHNESS_PLAN_FAILED|LINE=124|if kind=="VM":
REC|AUDIT_SOURCE|CODE=APP_FRESHNESS_PLAN_FAILED|LINE=125|typ=str(resources.get(ident,{}).get("type","qemu"))
REC|AUDIT_SOURCE|CODE=APP_FRESHNESS_PLAN_FAILED|LINE=126|sid=("ct"+ident) if typ=="lxc" else ("vm"+ident)
REC|AUDIT_SOURCE|CODE=APP_FRESHNESS_PLAN_FAILED|LINE=127|v2=None
REC|AUDIT_SOURCE|CODE=APP_FRESHNESS_PLAN_FAILED|LINE=128|try: v2=json.loads((STATE/(sid+".json")).read_text(encoding="utf-8"))
AUDIT_SOURCE_GFS_INVALID_HIT_COUNT=1
REC|AUDIT_SOURCE|CODE=GFS_INVALID|LINE=207|if lock=="PATH_REDACTED and expected_v2_heavy_lock_owner():
REC|AUDIT_SOURCE|CODE=GFS_INVALID|LINE=208|pass
REC|AUDIT_SOURCE|CODE=GFS_INVALID|LINE=209|else:
REC|AUDIT_SOURCE|CODE=GFS_INVALID|LINE=210|issues.append({"severity":"HIGH","code":"LOCK_BUSY","logical_id":lock,"root_cause":"ACTIVE_OR_ORPHAN_LOCK"})
REC|AUDIT_SOURCE|CODE=GFS_INVALID|LINE=211|
REC|AUDIT_SOURCE|CODE=GFS_INVALID|LINE=212|env=os.environ.copy(); env["HOMELAB_BACKUP_GFS_POLICY"]=GFSP; env["HOMELAB_BACKUP_TARGET_REGISTRY"]=str(REG)
REC|AUDIT_SOURCE|CODE=GFS_INVALID|LINE=213|grc,gout,_=run([GFSH,"validate"],20,env)
REC|AUDIT_SOURCE|CODE=GFS_INVALID|LINE=214|if grc!=0 or "GFS_POLICY=OK" not in gout:
REC|AUDIT_SOURCE|CODE=GFS_INVALID|LINE=215|issues.append({"severity":"CRITICAL","code":"GFS_INVALID","logical_id":"-","root_cause":"GFS_VALIDATE_FAILED"})
REC|AUDIT_SOURCE|CODE=GFS_INVALID|LINE=216|rr,_,_=run([RET,"contract"],120)
REC|AUDIT_SOURCE|CODE=GFS_INVALID|LINE=217|if rr!=0: issues.append({"severity":"HIGH","code":"RETENTION_CONTRACT_BAD","logical_id":"-","root_cause":"RETENTION_CONTRACT_RC_"+str(rr)})
REC|AUDIT_SOURCE|CODE=GFS_INVALID|LINE=218|rhp=pathlib.Path("PATH_REDACTED
REC|AUDIT_SOURCE|CODE=GFS_INVALID|LINE=219|if not rhp.is_file():
REC|AUDIT_SOURCE|CODE=GFS_INVALID|LINE=220|issues.append({"severity":"HIGH","code":"RETENTION_HEALTH_MISSING","logical_id":"-","root_cause":"RETENTION_HEALTH_FILE_MISSING"})
REC|AUDIT_SOURCE|CODE=GFS_INVALID|LINE=221|else:
REC|AUDIT_SOURCE|CODE=GFS_INVALID|LINE=222|try:
REC|AUDIT_SOURCE|CODE=GFS_INVALID|LINE=223|rh=json.loads(rhp.read_text(encoding="utf-8"))
AUDIT_SOURCE_RETENTION_CONTRACT_BAD_HIT_COUNT=1
REC|AUDIT_SOURCE|CODE=RETENTION_CONTRACT_BAD|LINE=209|else:
REC|AUDIT_SOURCE|CODE=RETENTION_CONTRACT_BAD|LINE=210|issues.append({"severity":"HIGH","code":"LOCK_BUSY","logical_id":lock,"root_cause":"ACTIVE_OR_ORPHAN_LOCK"})
REC|AUDIT_SOURCE|CODE=RETENTION_CONTRACT_BAD|LINE=211|
REC|AUDIT_SOURCE|CODE=RETENTION_CONTRACT_BAD|LINE=212|env=os.environ.copy(); env["HOMELAB_BACKUP_GFS_POLICY"]=GFSP; env["HOMELAB_BACKUP_TARGET_REGISTRY"]=str(REG)
REC|AUDIT_SOURCE|CODE=RETENTION_CONTRACT_BAD|LINE=213|grc,gout,_=run([GFSH,"validate"],20,env)
REC|AUDIT_SOURCE|CODE=RETENTION_CONTRACT_BAD|LINE=214|if grc!=0 or "GFS_POLICY=OK" not in gout:
REC|AUDIT_SOURCE|CODE=RETENTION_CONTRACT_BAD|LINE=215|issues.append({"severity":"CRITICAL","code":"GFS_INVALID","logical_id":"-","root_cause":"GFS_VALIDATE_FAILED"})
REC|AUDIT_SOURCE|CODE=RETENTION_CONTRACT_BAD|LINE=216|rr,_,_=run([RET,"contract"],120)
REC|AUDIT_SOURCE|CODE=RETENTION_CONTRACT_BAD|LINE=217|if rr!=0: issues.append({"severity":"HIGH","code":"RETENTION_CONTRACT_BAD","logical_id":"-","root_cause":"RETENTION_CONTRACT_RC_"+str(rr)})
REC|AUDIT_SOURCE|CODE=RETENTION_CONTRACT_BAD|LINE=218|rhp=pathlib.Path("PATH_REDACTED
REC|AUDIT_SOURCE|CODE=RETENTION_CONTRACT_BAD|LINE=219|if not rhp.is_file():
REC|AUDIT_SOURCE|CODE=RETENTION_CONTRACT_BAD|LINE=220|issues.append({"severity":"HIGH","code":"RETENTION_HEALTH_MISSING","logical_id":"-","root_cause":"RETENTION_HEALTH_FILE_MISSING"})
REC|AUDIT_SOURCE|CODE=RETENTION_CONTRACT_BAD|LINE=221|else:
REC|AUDIT_SOURCE|CODE=RETENTION_CONTRACT_BAD|LINE=222|try:
REC|AUDIT_SOURCE|CODE=RETENTION_CONTRACT_BAD|LINE=223|rh=json.loads(rhp.read_text(encoding="utf-8"))
REC|AUDIT_SOURCE|CODE=RETENTION_CONTRACT_BAD|LINE=224|if rh.get("status")!="OK": issues.append({"severity":"HIGH","code":"RETENTION_HEALTH_BAD","logical_id":"-","root_cause":"RETENTION_HEALTH_NOT_OK"})
REC|AUDIT_SOURCE|CODE=RETENTION_CONTRACT_BAD|LINE=225|except Exception: issues.append({"severity":"HIGH","code":"RETENTION_HEALTH_BAD","logical_id":"-","root_cause":"RETENTION_HEALTH_UNREADABLE"})
REC|AUDIT_PATH_REF|BASENAME=targets.json|PARENT=homelab-backup|TYPE=file|SHA256=c588c4aa0c941406efcb3d24876c9262d9ec0dd4ced297d456fd2193e83afc49|PATH_SHA=2f97838a94d9466163297d47d72c9c2e903a12cfb344abc58ff33fcbbae05528
REC|AUDIT_PATH_REF|BASENAME=state|PARENT=homelab-backup-v2|TYPE=dir|SHA256=NONE|PATH_SHA=6b3144b23b95313197d83e8d55533ff96cc06062af4ae20c3d7a635cdc56145e
REC|AUDIT_PATH_REF|BASENAME=cloud-quorum|PARENT=homelab-backup|TYPE=dir|SHA256=NONE|PATH_SHA=8757ada57c0d5e9d3c7d43ff9f4463f9cd391a8f19c6952b0c872e10a305630c
REC|AUDIT_PATH_REF|BASENAME=vm-restore-validation|PARENT=homelab-backup|TYPE=dir|SHA256=NONE|PATH_SHA=c56996e4ea53994a88601ceeed1e5291c3e7ea79661386090cf582c3036616f8
REC|AUDIT_PATH_REF|BASENAME=app-restore-validation|PARENT=homelab-backup|TYPE=dir|SHA256=NONE|PATH_SHA=c70a6d9e128e0f6fc7e29453ab78510eb2ff57124d19767bd88f87659063751c
REC|AUDIT_PATH_REF|BASENAME=homelab-backup-target-registry|PARENT=libexec|TYPE=file|SHA256=66a243a207cbc5e19255fcafdb06013008d1d8c1d1205e660fdf0f236ff6a15a|PATH_SHA=cd6e91b27939aee60ecf8f93a42b3c793926bfd8930593f3ae3bb95f72692162
REC|AUDIT_PATH_REF|BASENAME=homelab-backup-gfs-policy|PARENT=libexec|TYPE=file|SHA256=a1aed0c6039f8095bd5da7f62b47f4ee6932df98c4e0e7420d3006df86bd5c94|PATH_SHA=d731f7b73d3dd86c74980bed86852aea70b61074102fc4792f958096a43441a9
REC|AUDIT_PATH_REF|BASENAME=gfs-retention.json|PARENT=homelab-backup|TYPE=file|SHA256=2c7674045dd5c68cc48d348d616494dbbe145df460a8e85d97a2c5c153ec792d|PATH_SHA=e76524b2cd2eff155d83a860d2ff2c65c0b54695eb6445ecc2af88cfbf32816d
REC|AUDIT_PATH_REF|BASENAME=homelab-backup-retention-gc|PARENT=sbin|TYPE=file|SHA256=a96e607107e7b4e7762f7cb7f0e91f8c3ec6b793ed4604633c125433839ac909|PATH_SHA=68bcf5e97e4eda63f88bfee3ece5bc50ccdcbb2a089f97923534e67daf29ee4a
REC|AUDIT_PATH_REF|BASENAME=restore-check-health.txt|PARENT=skladchik-reports-monitor-backup|TYPE=file|SHA256=08a290bea388e4ca0e1160aab1c909ff0cef9e98326b6dd32abe9f5541125ce8|PATH_SHA=c7b01e883c50df0f85e999374fa9d383932f47207b7a088028bbbd3fae148509
REC|AUDIT_PATH_REF|BASENAME=backup-audit-v2.json|PARENT=homelab-health|TYPE=file|SHA256=2ece5343c5d732cbaa65d16feafaade791a2eb2eae465d8a6aba4e644c12e49a|PATH_SHA=e520cdf4c0a133ac68165ccf96f4bec1031f4d7a3e7bddccf5c84aea2a26cad7
REC|AUDIT_PATH_REF|BASENAME=backup-audit.json|PARENT=homelab-health|TYPE=file|SHA256=2ece5343c5d732cbaa65d16feafaade791a2eb2eae465d8a6aba4e644c12e49a|PATH_SHA=091a828db6ee335d0272a370019f274a88f3a0fa4c9fd92d268358a3674f0b1d
REC|AUDIT_PATH_REF|BASENAME=homelab-backup-heavy.lock|PARENT=run|TYPE=file|SHA256=e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855|PATH_SHA=2642241b0beaa7376e1362beefa000e2137478b93e8c1a5d275189b4954a6c93
REC|AUDIT_PATH_REF|BASENAME=homelab-backup-heavy.lock|PARENT=run|TYPE=file|SHA256=e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855|PATH_SHA=2642241b0beaa7376e1362beefa000e2137478b93e8c1a5d275189b4954a6c93
REC|AUDIT_PATH_REF|BASENAME=homelab-backup-dispatcher.lock|PARENT=lock|TYPE=file|SHA256=e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855|PATH_SHA=448d0e8fe73ac64db817c9df04aede92c9cea524652755a9fcdf60fbad00c1aa
REC|AUDIT_PATH_REF|BASENAME=homelab-backup-heavy.lock|PARENT=run|TYPE=file|SHA256=e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855|PATH_SHA=2642241b0beaa7376e1362beefa000e2137478b93e8c1a5d275189b4954a6c93
REC|AUDIT_PATH_REF|BASENAME=backup-retention-safe-gc.json|PARENT=homelab-health|TYPE=file|SHA256=4260fcbc5797375b9281e9c64fec35b203fcd35110395e10b94a602a2a67b48c|PATH_SHA=9427dedc6832eded4f955d7a1b6251430298b78c17ed2970664df3370485c719
REC|AUDIT_PATH_REF|BASENAME=app-skladchik-reports-monitor|PARENT=cloud-quorum|TYPE=dir|SHA256=NONE|PATH_SHA=dc50a937c87b10fe436df9da05826c17b7a3d17d316d87035e5571051c8358a3
REC|AUDIT_PATH_REF|BASENAME=app-skladchik-reports-monitor|PARENT=app-restore-validation|TYPE=dir|SHA256=NONE|PATH_SHA=c75d5957fa9c11e592017a8399ae56adcdb992ade172a54162a8177c1291f45a
BACKUP_AUDIT_DIRECT_RC=1
REC|AUDIT_DIRECT|ISSUE|severity=CRITICAL|code=APP_FRESHNESS_PLAN_FAILED|logical_id=-|root_cause=APP_QUEUE_RC_65
REC|AUDIT_DIRECT|ISSUE|severity=CRITICAL|code=GFS_INVALID|logical_id=-|root_cause=GFS_VALIDATE_FAILED
REC|AUDIT_DIRECT|ISSUE|severity=HIGH|code=RETENTION_CONTRACT_BAD|logical_id=-|root_cause=RETENTION_CONTRACT_RC_65
REC|AUDIT_DIRECT|SUMMARY|AUDIT_STATUS=OK|BACKUP_STATUS=RED|ACTIVE_COUNT=15|ISSUE_COUNT=3|CLOUD_A=OK|CLOUD_B=OK|CLOUD_B_DRILL=OK|DR_PROVEN=YES|SOURCE_OFFHOST=OK
BACKUP_AUDIT_DIRECT_STDERR_LINES=0
VPN_EXEC_PRESENT=true
VPN_EXEC_BASENAME=bash
VPN_EXEC_SHA=223fe8564b60636bc738b6178d3ba9a50ef7d791266b0efae6363bb716e4c47f
VPN_RESULT_NOW=exit-code
VPN_EXEC_STATUS_NOW=1
REC|VPN_JOURNAL|1|Starting homelab-private-vpn-hosts-health.service - Homelab private VPN hosts health check...
REC|VPN_JOURNAL|2|homelab-private-vpn-hosts-health.service: Main process exited, code=exited, status=1/FAILURE
REC|VPN_JOURNAL|3|homelab-private-vpn-hosts-health.service: Failed with result 'exit-code'.
REC|VPN_JOURNAL|4|Failed to start homelab-private-vpn-hosts-health.service - Homelab private VPN hosts health check.
REC|VPN_JOURNAL|5|Starting homelab-private-vpn-hosts-health.service - Homelab private VPN hosts health check...
REC|VPN_JOURNAL|6|homelab-private-vpn-hosts-health.service: Main process exited, code=exited, status=1/FAILURE
REC|VPN_JOURNAL|7|homelab-private-vpn-hosts-health.service: Failed with result 'exit-code'.
REC|VPN_JOURNAL|8|Failed to start homelab-private-vpn-hosts-health.service - Homelab private VPN hosts health check.
REC|VPN_JOURNAL|9|Starting homelab-private-vpn-hosts-health.service - Homelab private VPN hosts health check...
REC|VPN_JOURNAL|10|homelab-private-vpn-hosts-health.service: Main process exited, code=exited, status=1/FAILURE
REC|VPN_JOURNAL|11|homelab-private-vpn-hosts-health.service: Failed with result 'exit-code'.
REC|VPN_JOURNAL|12|Failed to start homelab-private-vpn-hosts-health.service - Homelab private VPN hosts health check.
REC|VPN_JOURNAL|13|Starting homelab-private-vpn-hosts-health.service - Homelab private VPN hosts health check...
REC|VPN_JOURNAL|14|homelab-private-vpn-hosts-health.service: Main process exited, code=exited, status=1/FAILURE
REC|VPN_JOURNAL|15|homelab-private-vpn-hosts-health.service: Failed with result 'exit-code'.
REC|VPN_JOURNAL|16|Failed to start homelab-private-vpn-hosts-health.service - Homelab private VPN hosts health check.
REC|VPN_JOURNAL|17|Starting homelab-private-vpn-hosts-health.service - Homelab private VPN hosts health check...
REC|VPN_JOURNAL|18|homelab-private-vpn-hosts-health.service: Main process exited, code=exited, status=1/FAILURE
REC|VPN_JOURNAL|19|homelab-private-vpn-hosts-health.service: Failed with result 'exit-code'.
REC|VPN_JOURNAL|20|Failed to start homelab-private-vpn-hosts-health.service - Homelab private VPN hosts health check.
REC|VPN_JOURNAL|21|Starting homelab-private-vpn-hosts-health.service - Homelab private VPN hosts health check...
REC|VPN_JOURNAL|22|homelab-private-vpn-hosts-health.service: Main process exited, code=exited, status=1/FAILURE
REC|VPN_JOURNAL|23|homelab-private-vpn-hosts-health.service: Failed with result 'exit-code'.
REC|VPN_JOURNAL|24|Failed to start homelab-private-vpn-hosts-health.service - Homelab private VPN hosts health check.
REC|VPN_JOURNAL|25|Starting homelab-private-vpn-hosts-health.service - Homelab private VPN hosts health check...
REC|VPN_JOURNAL|26|homelab-private-vpn-hosts-health.service: Main process exited, code=exited, status=1/FAILURE
REC|VPN_JOURNAL|27|homelab-private-vpn-hosts-health.service: Failed with result 'exit-code'.
REC|VPN_JOURNAL|28|Failed to start homelab-private-vpn-hosts-health.service - Homelab private VPN hosts health check.
REC|VPN_JOURNAL|29|Starting homelab-private-vpn-hosts-health.service - Homelab private VPN hosts health check...
REC|VPN_JOURNAL|30|homelab-private-vpn-hosts-health.service: Main process exited, code=exited, status=1/FAILURE
REC|VPN_JOURNAL|31|homelab-private-vpn-hosts-health.service: Failed with result 'exit-code'.
REC|VPN_JOURNAL|32|Failed to start homelab-private-vpn-hosts-health.service - Homelab private VPN hosts health check.
REC|VPN_JOURNAL|33|Starting homelab-private-vpn-hosts-health.service - Homelab private VPN hosts health check...
REC|VPN_JOURNAL|34|homelab-private-vpn-hosts-health.service: Main process exited, code=exited, status=1/FAILURE
REC|VPN_JOURNAL|35|homelab-private-vpn-hosts-health.service: Failed with result 'exit-code'.
REC|VPN_JOURNAL|36|Failed to start homelab-private-vpn-hosts-health.service - Homelab private VPN hosts health check.
REC|VPN_JOURNAL|37|Starting homelab-private-vpn-hosts-health.service - Homelab private VPN hosts health check...
REC|VPN_JOURNAL|38|homelab-private-vpn-hosts-health.service: Main process exited, code=exited, status=1/FAILURE
REC|VPN_JOURNAL|39|homelab-private-vpn-hosts-health.service: Failed with result 'exit-code'.
REC|VPN_JOURNAL|40|Failed to start homelab-private-vpn-hosts-health.service - Homelab private VPN hosts health check.
VPN_EXPECTED_NAME_COUNT=0
VPN_PVE01_RESOLV_CONF_SHA=a89871ac2938a423c74c6dfe801eab57085b4a4c869bb1f864097f5885e27a74
VPN_PVE01_RESOLV_NAMESERVER_COUNT=1
VPN_DIRECT_RC=0
REC|VPN_DIRECT_OUT|1|PVE2_VPN_UNIT_EXISTS=false
REC|VPN_DIRECT_OUT|2|PVE2_VPN_RESULT=success
REC|VPN_DIRECT_OUT|3|PVE2_RESOLV_SHA=466da4d9381fbd060c635244fa97a800e50b77613fbd8c2ea0d2057ad4617f1f
REC|VPN_DIRECT_OUT|4|PVE2_RESOLV_NAMESERVER_COUNT=1
REC|VPN_DIRECT_OUT|5|FAILED_UNIT_COUNT_NOW=2
REC|VPN_DIRECT_OUT|6|REC|FAILED_UNIT_NOW|homelab-backup-audit.service
REC|VPN_DIRECT_OUT|7|REC|FAILED_UNIT_NOW|homelab-private-vpn-hosts-health.service
REC|VPN_DIRECT_OUT|8|APT_FULL_UPGRADE_SIM_RC=0
REC|VPN_DIRECT_OUT|9|APT_FULL_UPGRADE_SIM_UPGRADE_COUNT=184
REC|VPN_DIRECT_OUT|10|APT_FULL_UPGRADE_SIM_REMOVE_COUNT=0
REC|VPN_DIRECT_OUT|11|DECISION=PASS_PVE01_REMAINING_FAILED_UNITS_RCA
REC|VPN_DIRECT_OUT|12|NEXT_GATE=PVE01_REMAINING_FAILED_UNITS_REPAIR
REC|VPN_DIRECT_OUT|13|TASK_RESULT=PVE01_REMAINING_FAILED_UNITS_RCA
REC|VPN_DIRECT_OUT|14|CHANGES_MADE=false
REC|VPN_DIRECT_OUT|15|ROLLBACK_STARTED=false
REC|VPN_DIRECT_OUT|16|ROLLBACK_RESTORED=null
REC|VPN_DIRECT_OUT|17|MUTATION_OUTCOME=NO_MUTATION
REC|VPN_DIRECT_OUT|18|HOMELAB_RESULT_CONTRACT={"version":1,"command_id":"SUPPORT-261006-PVE01-REMAINING-FAILED-UNITS-RCA-99694R1","status":"OK","changes_made":false,"rollback_started":false,"rollback_restored":null}
OUTPUT_END
CHAT_OUTPUT_END