From 4d1be35ed5551320946d056063c00d74aefe0d4d Mon Sep 17 00:00:00 2001
From: Anthony Wen <anthony.wen@cdsi.us.com>
Date: Mon, 3 Aug 2026 22:50:04 -0400
Subject: [PATCH] Fix ATVM watcher service restart after failed units

---
 atvm/watcher-service/README.md                 | 1 +
 atvm/watcher-service/start-atvm-run-watcher.sh | 1 +
 atvm/watcher-service/start-atvm-runner.sh      | 1 +
 skills/atvm/SKILL.md                           | 7 +++++++
 4 files changed, 10 insertions(+)

diff --git a/atvm/watcher-service/README.md b/atvm/watcher-service/README.md
index 81ee67d..0e8d92d 100644
--- a/atvm/watcher-service/README.md
+++ b/atvm/watcher-service/README.md
@@ -181,6 +181,7 @@ Runner cancel example:
 - The watcher uses the same ATVM status layout documented in `atvm/docs/automation/status-template.md`.
 - Prefer the controller-local `atvm-runner@...` service over ad hoc `nohup` or detached SSH launch patterns for `run-sorry-cypress.py`.
 - Prefer `start-atvm-run.sh` when launching both services together because it prevents the watcher/runner log-path race by enforcing watcher-first ordering.
+- The start helpers clear stale failed systemd state with `systemctl reset-failed` before starting per-build watcher/runner units.
 - Kernel values are resolved from `atvm/inventory/vm-inventory.md`.
 - Categorized execution is treated as sequential grouped ATVM sub-runs, not as one parent run with internal phases.
 - In categorized mode, the watcher writes per-subrun state under `subruns/` and posts each completed grouped run separately.
diff --git a/atvm/watcher-service/start-atvm-run-watcher.sh b/atvm/watcher-service/start-atvm-run-watcher.sh
index 3fa54d6..b8a412a 100644
--- a/atvm/watcher-service/start-atvm-run-watcher.sh
+++ b/atvm/watcher-service/start-atvm-run-watcher.sh
@@ -63,6 +63,7 @@ fi
 RUN_DIR="${STATE_ROOT}/${BUILD_NAME}"
 RUN_LOG="/tmp/${BUILD_NAME}.log"
 systemctl stop "atvm-run-watcher@${BUILD_NAME}.service" >/dev/null 2>&1 || true
+systemctl reset-failed "atvm-run-watcher@${BUILD_NAME}.service" >/dev/null 2>&1 || true
 rm -rf "$RUN_DIR"
 rm -f "$RUN_LOG"
 mkdir -p "$RUN_DIR"
diff --git a/atvm/watcher-service/start-atvm-runner.sh b/atvm/watcher-service/start-atvm-runner.sh
index 63012bd..1a90abf 100644
--- a/atvm/watcher-service/start-atvm-runner.sh
+++ b/atvm/watcher-service/start-atvm-runner.sh
@@ -59,5 +59,6 @@ ATVM_RUNNER_LOG=${RUNNER_LOG@Q}
 EOF
 
 systemctl stop "atvm-runner@${BUILD_NAME}.service" >/dev/null 2>&1 || true
+systemctl reset-failed "atvm-runner@${BUILD_NAME}.service" >/dev/null 2>&1 || true
 systemctl start "atvm-runner@${BUILD_NAME}.service"
 systemctl status --no-pager "atvm-runner@${BUILD_NAME}.service" || true
diff --git a/skills/atvm/SKILL.md b/skills/atvm/SKILL.md
index 9b78e2d..38ba3fa 100644
--- a/skills/atvm/SKILL.md
+++ b/skills/atvm/SKILL.md
@@ -206,6 +206,13 @@ Typical controller-side tools:
 - `cmc-templates.py`
 - `run-sorry-cypress.py`
 
+Default controller watcher wrapper:
+- `/opt/atvm-watcher-service/start-atvm-run.sh`
+
+Use that wrapper for watcher-backed ATVM automation launches when present. Do
+not search only inside the Cypress checkout for the wrapper; on the ATVM
+Cypress controller it is installed under `/opt/atvm-watcher-service`.
+
 Treat their exact location as environment-specific unless a local repo or controller path is available.
 
 ## Artifact Recovery and Reporting