fix atvm watcher-backed run launch sequence

Execute the template step before starting watcher-backed ATVM runs.

- run --template-command synchronously in start-atvm-run.sh
- write template output to /tmp/<build>.launch.log
- stop before watcher/runner startup if template generation fails
- document the corrected wrapper behavior in watcher-service docs
- record the stale specPattern failure mode in automation run learnings

atvm/watcher-service/start-atvm-run.sh

@@ -37,6 +37,7 @@ SCOPE_DESCRIPTION=""
 WATCHER_CATEGORIZED="false"
 RUNNER_WORKDIR="/root/cdc-e2e-cyp-12.17.4"
 RUNNER_LOG=""
+LAUNCH_LOG=""
 STATE_ROOT="/var/lib/atvm-run-watcher"
 
 while [[ $# -gt 0 ]]; do
@@ -76,6 +77,8 @@ if [[ -z "$RUNNER_LOG" ]]; then
   RUNNER_LOG="/tmp/${BUILD_NAME}.log"
 fi
 
+LAUNCH_LOG="/tmp/${BUILD_NAME}.launch.log"
+
 SCRIPT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)"
 
 watcher_cmd=(
@@ -112,6 +115,25 @@ runner_cmd=(
   --state-root "${STATE_ROOT}"
 )
 
+mkdir -p "$(dirname "${LAUNCH_LOG}")"
+: > "${LAUNCH_LOG}"
+
+if [[ -n "${TEMPLATE_COMMAND}" ]]; then
+  {
+    echo "Running template command:"
+    echo "${TEMPLATE_COMMAND}"
+    echo
+  } >>"${LAUNCH_LOG}"
+
+  if ! (
+    cd "${RUNNER_WORKDIR}"
+    bash -lc "${TEMPLATE_COMMAND}"
+  ) >>"${LAUNCH_LOG}" 2>&1; then
+    echo "Template command failed for ${BUILD_NAME}. See ${LAUNCH_LOG}" >&2
+    exit 1
+  fi
+fi
+
 "${watcher_cmd[@]}"
 
 for _ in {1..15}; do