diff --git a/atvm/docs/automation/run-learnings.md b/atvm/docs/automation/run-learnings.md index 48131c4..bd44cc8 100644 --- a/atvm/docs/automation/run-learnings.md +++ b/atvm/docs/automation/run-learnings.md @@ -487,3 +487,13 @@ This file stores run-specific examples only when a run produced a new learning r - Keep `HOSTS` detail to the failing step plus a short error summary only. - Put richer per-host error excerpts in `FAILURE NOTES:`. - Reserve `NOTES:` for non-failure context such as template command, Currents URL, and operator-facing caveats. + +## Run Learning: 2026-03-31 (Mochawesome failure parsing must stay within one testcase object) +- Observed failure mode: + - A reboot failure post showed step `36` with an empty `FAILURE NOTES:` excerpt even though the real failure remained step `38` and the mochawesome HTML contained the full `sshpass` / `md5sum` error text. + - The parser was scanning beyond the current mochawesome testcase object, so it paired one step title with another step's later failed-state/message fields. + - Empty mochawesome `message` / `estack` values must not be accepted as valid failure detail. +- Action for future runs: + - Parse mochawesome one testcase object at a time and do not cross object boundaries when matching `title`, `fullTitle`, `state`, `message`, and `estack`. + - Only use mochawesome to enrich host detail when it returns a non-empty failure payload. + - If mochawesome and structured reporter artifacts disagree on the step number, keep the structured reporter step as the safer fallback for the host detail. diff --git a/atvm/watcher-service/atvm_run_watcher.py b/atvm/watcher-service/atvm_run_watcher.py index 72a9782..b55f907 100644 --- a/atvm/watcher-service/atvm_run_watcher.py +++ b/atvm/watcher-service/atvm_run_watcher.py @@ -540,6 +540,20 @@ def extract_first_json_string(block: str, key: str) -> Optional[str]: return decode_json_string_fragment(match.group(1)) +def extract_step_number(text: str) -> Optional[int]: + match = re.search(r"\b(\d+)\.\s", text) + if not match: + return None + return int(match.group(1)) + + +def extract_testcase_from_host_detail(detail: str) -> Optional[str]: + match = re.match(r"^\d+ tests, \d+ failures(?:, \d+ pending)? - (.+?)(?: - .+)?$", detail) + if not match: + return None + return match.group(1).strip() + + def extract_failure_from_mochawesome( reporter_root: Path, build_name: str, @@ -565,15 +579,28 @@ def extract_failure_from_mochawesome( full_title = decode_json_string_fragment(match.group(1)) if host not in full_title: continue - block_start = max(0, match.start() - 1200) - block_end = min(len(text), match.end() + 8000) - block = text[block_start:block_end] - if not re.search(r'"state":"failed"', block): + object_start = text.rfind('{"title":"', 0, match.start()) + alternate_start = text.rfind(',"title":"', 0, match.start()) + if alternate_start != -1 and (object_start == -1 or alternate_start > object_start): + object_start = alternate_start + 1 + if object_start == -1: continue - testcase = extract_first_json_string(block, "title") or "failed testcase" - message = extract_first_json_string(block, "message") or "" - estack = extract_first_json_string(block, "estack") or "" - if testcase or message or estack: + object_end_candidates = [ + index for index in ( + text.find('},{"title":"', match.end()), + text.find('}]}', match.end()), + text.find('}]}', match.end()), + ) + if index != -1 + ] + object_end = min(object_end_candidates) if object_end_candidates else min(len(text), match.end() + 16000) + obj = text[object_start:object_end] + if not re.search(r'"state":"failed"', obj): + continue + testcase = extract_first_json_string(obj, "title") or "failed testcase" + message = extract_first_json_string(obj, "message") or "" + estack = extract_first_json_string(obj, "estack") or "" + if testcase and (message or estack): return testcase, message, estack return None @@ -584,7 +611,7 @@ def summarize_host_detail_with_mochawesome(detail: str, testcase: str, message: normalized_message = " ".join((message or "").split()) message_summary = "" for pattern in ( - r"(md5sum:\s.*?)(?:$)", + r"(md5sum:\s.*?No such file or directory)", r"(sshpass does not contain OK(?:\.\s*Output:)?)(?:$)", r"(AssertionError:\s.*?)(?:$)", r"(Timed out!? .*?)(?:$)", @@ -1203,11 +1230,22 @@ def build_status_markdown( if not mochawesome_failure: continue testcase, message, estack = mochawesome_failure + if not (message or estack): + continue + existing_step = extract_step_number(host.detail) + mochawesome_step = extract_step_number(testcase) + if ( + existing_step is not None + and mochawesome_step is not None + and existing_step != mochawesome_step + ): + testcase = extract_testcase_from_host_detail(host.detail) or testcase host.detail = summarize_host_detail_with_mochawesome(host.detail, testcase, message) failure_excerpt_source = estack or message - failure_notes.append( - f"{host.host} failure excerpt: `{compact_failure_detail(failure_excerpt_source, limit=420)}`" - ) + if failure_excerpt_source.strip(): + failure_notes.append( + f"{host.host} failure excerpt: `{compact_failure_detail(failure_excerpt_source, limit=420)}`" + ) host_lines = ["| Host | Kernel | Status | Detail |", "| --- | --- | --- | --- |"] for host in ordered_hosts: