From 5f7824619d776c8ecf01b0ba902e049cc9da12ba Mon Sep 17 00:00:00 2001 From: Anthony Wen Date: Sat, 13 Jun 2026 18:41:19 -0400 Subject: [PATCH] Simplify ATVM host detail reporting --- atvm/docs/automation/guide.md | 2 ++ atvm/docs/automation/run-learnings.md | 9 +++++++++ atvm/watcher-service/atvm_run_watcher.py | 24 +++++++++++++++++++++++- 3 files changed, 34 insertions(+), 1 deletion(-) diff --git a/atvm/docs/automation/guide.md b/atvm/docs/automation/guide.md index 0e53802..974e38b 100644 --- a/atvm/docs/automation/guide.md +++ b/atvm/docs/automation/guide.md @@ -315,6 +315,8 @@ Status-report expectations: - Do not include generic watcher bookkeeping messages in `NOTES:` such as artifact-detection confirmations. - Do not include internal watcher fallback notes in `NOTES:` such as `check-xml-files.ts` validation confirmations or reporter-artifact recovery details. - The `HOSTS:` table includes `Host`, `Kernel`, `Status`, and `Detail` columns in that order. +- Do not show total test/failure counts in the `Detail` column. Keep those counts in watcher state and summary logic only. +- For passed hosts, render `Detail` as `completed`. - For any failed host, keep the `Detail` column compact by showing the failing step plus a short error summary, not the full raw stack trace. - If richer failure text is available, put the longer trimmed excerpt in `FAILURE NOTES:` so the result stays readable in Mattermost and local status output. - In `COVERAGE:`, describe the important `cmc-templates.py` command inputs such as template, categorize mode, datastore/config family, config filename, migration style, any real plugin/integration path, and other operator-relevant run options, but do not list target hosts there or include verbose prose scope descriptions. diff --git a/atvm/docs/automation/run-learnings.md b/atvm/docs/automation/run-learnings.md index 5cfcc90..f7f2927 100644 --- a/atvm/docs/automation/run-learnings.md +++ b/atvm/docs/automation/run-learnings.md @@ -665,3 +665,12 @@ This file stores run-specific examples only when a run produced a new learning r - Action for future runs: - When a run is marked `FAILED` from hang-kill markers or non-zero runner exit and no host results are available, synthesize one failed host row from current host/spec inference. - Use a clear failure detail such as `hang timeout killed runner` so operator-facing status always includes a concrete host failure line. + +## Run Learning: 2026-06-04 (Host detail should not show total test counts) +- Observed failure mode: + - A passing compute-migration run posted `29 tests, 0 failures` in the `HOSTS` detail column while `TEST FLOW:` listed 30 planned/generated spec steps. + - The mismatch was confusing because host detail came from actual Cypress reporter counts, while `TEST FLOW:` came from static generated-spec extraction. +- Action for future runs: + - Keep raw test/failure counts in watcher state for classification and debugging, but do not render them in the `HOSTS` detail column. + - For passing hosts, render detail as `completed`. + - For failed hosts, render only the failing step plus compact error summary; put richer excerpts in `FAILURE NOTES:`. diff --git a/atvm/watcher-service/atvm_run_watcher.py b/atvm/watcher-service/atvm_run_watcher.py index 4d88137..189b328 100644 --- a/atvm/watcher-service/atvm_run_watcher.py +++ b/atvm/watcher-service/atvm_run_watcher.py @@ -711,6 +711,28 @@ def summarize_host_detail_with_mochawesome(detail: str, testcase: str, message: return f"{prefix} - {testcase_summary} - {message_summary}" +def display_host_detail(host: HostResult) -> str: + detail = (host.detail or "").strip() + if host.status == "PASS": + return "completed" + if host.status in {"RUN", "NOT STARTED"}: + return "in progress" + if host.status == "FAIL": + detail = re.sub( + r"^\d+\s+tests?,\s+\d+\s+failures?(?:,\s+\d+\s+pending)?\s*-\s*", + "", + detail, + flags=re.I, + ) + detail = re.sub(r"^\d+\s+failures?\s*-\s*", "", detail, flags=re.I) + if re.fullmatch(r"\d+\s+tests?,\s+\d+\s+failures?(?:,\s+\d+\s+pending)?", detail, flags=re.I): + return "failed" + if re.fullmatch(r"\d+\s+failures?", detail, flags=re.I): + return "failed" + return detail or "failed" + return detail + + def extract_host_results_from_run_finished_segment(segment_text: str, inventory: Dict[str, str]) -> Dict[str, HostResult]: host_results: Dict[str, HostResult] = {} # Currents can wrap the trailing "s" in long duration cells onto its own table row. @@ -1536,7 +1558,7 @@ def build_status_markdown( "SKIP": "⏭️ SKIP", "NOT STARTED": "⏳ RUN", }.get(host.status, host.status) - host_lines.append(f"| {host.host} | {host.kernel} | {icon} | {host.detail} |") + host_lines.append(f"| {host.host} | {host.kernel} | {icon} | {display_host_detail(host)} |") if currents_url: notes = notes + [f"Currents recorded run: `{currents_url}`"]