Fix ATVM mochawesome failure extraction

atvm/docs/automation/run-learnings.md

@@ -487,3 +487,13 @@ This file stores run-specific examples only when a run produced a new learning r
   - Keep `HOSTS` detail to the failing step plus a short error summary only.
   - Put richer per-host error excerpts in `FAILURE NOTES:`.
   - Reserve `NOTES:` for non-failure context such as template command, Currents URL, and operator-facing caveats.
+
+## Run Learning: 2026-03-31 (Mochawesome failure parsing must stay within one testcase object)
+- Observed failure mode:
+  - A reboot failure post showed step `36` with an empty `FAILURE NOTES:` excerpt even though the real failure remained step `38` and the mochawesome HTML contained the full `sshpass` / `md5sum` error text.
+  - The parser was scanning beyond the current mochawesome testcase object, so it paired one step title with another step's later failed-state/message fields.
+  - Empty mochawesome `message` / `estack` values must not be accepted as valid failure detail.
+- Action for future runs:
+  - Parse mochawesome one testcase object at a time and do not cross object boundaries when matching `title`, `fullTitle`, `state`, `message`, and `estack`.
+  - Only use mochawesome to enrich host detail when it returns a non-empty failure payload.
+  - If mochawesome and structured reporter artifacts disagree on the step number, keep the structured reporter step as the safer fallback for the host detail.

atvm/watcher-service/atvm_run_watcher.py

@@ -540,6 +540,20 @@ def extract_first_json_string(block: str, key: str) -> Optional[str]:
     return decode_json_string_fragment(match.group(1))
 
 
+def extract_step_number(text: str) -> Optional[int]:
+    match = re.search(r"\b(\d+)\.\s", text)
+    if not match:
+        return None
+    return int(match.group(1))
+
+
+def extract_testcase_from_host_detail(detail: str) -> Optional[str]:
+    match = re.match(r"^\d+ tests, \d+ failures(?:, \d+ pending)? - (.+?)(?: - .+)?$", detail)
+    if not match:
+        return None
+    return match.group(1).strip()
+
+
 def extract_failure_from_mochawesome(
     reporter_root: Path,
     build_name: str,
@@ -565,15 +579,28 @@ def extract_failure_from_mochawesome(
             full_title = decode_json_string_fragment(match.group(1))
             if host not in full_title:
                 continue
-            block_start = max(0, match.start() - 1200)
-            block_end = min(len(text), match.end() + 8000)
-            block = text[block_start:block_end]
-            if not re.search(r'"state":"failed"', block):
+            object_start = text.rfind('{"title":"', 0, match.start())
+            alternate_start = text.rfind(',"title":"', 0, match.start())
+            if alternate_start != -1 and (object_start == -1 or alternate_start > object_start):
+                object_start = alternate_start + 1
+            if object_start == -1:
                 continue
-            testcase = extract_first_json_string(block, "title") or "failed testcase"
-            message = extract_first_json_string(block, "message") or ""
-            estack = extract_first_json_string(block, "estack") or ""
-            if testcase or message or estack:
+            object_end_candidates = [
+                index for index in (
+                    text.find('},{"title":"', match.end()),
+                    text.find('}]}', match.end()),
+                    text.find('}]}</script>', match.end()),
+                )
+                if index != -1
+            ]
+            object_end = min(object_end_candidates) if object_end_candidates else min(len(text), match.end() + 16000)
+            obj = text[object_start:object_end]
+            if not re.search(r'"state":"failed"', obj):
+                continue
+            testcase = extract_first_json_string(obj, "title") or "failed testcase"
+            message = extract_first_json_string(obj, "message") or ""
+            estack = extract_first_json_string(obj, "estack") or ""
+            if testcase and (message or estack):
                 return testcase, message, estack
     return None
 
@@ -584,7 +611,7 @@ def summarize_host_detail_with_mochawesome(detail: str, testcase: str, message:
     normalized_message = " ".join((message or "").split())
     message_summary = ""
     for pattern in (
-        r"(md5sum:\s.*?)(?:$)",
+        r"(md5sum:\s.*?No such file or directory)",
         r"(sshpass does not contain OK(?:\.\s*Output:)?)(?:$)",
         r"(AssertionError:\s.*?)(?:$)",
         r"(Timed out!? .*?)(?:$)",
@@ -1203,8 +1230,19 @@ def build_status_markdown(
         if not mochawesome_failure:
             continue
         testcase, message, estack = mochawesome_failure
+        if not (message or estack):
+            continue
+        existing_step = extract_step_number(host.detail)
+        mochawesome_step = extract_step_number(testcase)
+        if (
+            existing_step is not None
+            and mochawesome_step is not None
+            and existing_step != mochawesome_step
+        ):
+            testcase = extract_testcase_from_host_detail(host.detail) or testcase
         host.detail = summarize_host_detail_with_mochawesome(host.detail, testcase, message)
         failure_excerpt_source = estack or message
+        if failure_excerpt_source.strip():
             failure_notes.append(
                 f"{host.host} failure excerpt: `{compact_failure_detail(failure_excerpt_source, limit=420)}`"
             )